web search done right
MapReduce -> AMPC (#189)
Mikkel Denker3ab4f944e048
diff --git a/Cargo.lock b/Cargo.lockindex de7be79b..2046c11e 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -70,6 +70,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011"dependencies = ["cfg-if",+ "getrandom 0.2.12","once_cell","version_check","zerocopy",@@ -258,6 +259,15 @@ dependencies = ["syn 2.0.52",]+[[package]]+name = "atomic-polyfill"+version = "1.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8cf2bce30dfe09ef0bfaef228b9d414faaf7e563035494d7fe092dba54b300f4"+dependencies = [+ "critical-section",+]+[[package]]name = "attohttpc"version = "0.22.0"@@ -433,11 +443,20 @@ checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"[[package]]name = "bincode"-version = "1.3.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"+version = "2.0.0-rc.3"+source = "git+https://github.com/bincode-org/bincode?rev=8f65c60#8f65c60f192ff6093e6a1326b9d078a3b26c9a29"dependencies = [+ "bincode_derive","serde",+ "unty",+]++[[package]]+name = "bincode_derive"+version = "2.0.0-rc.3"+source = "git+https://github.com/bincode-org/bincode?rev=8f65c60#8f65c60f192ff6093e6a1326b9d078a3b26c9a29"+dependencies = [+ "virtue",][[package]]@@ -550,6 +569,28 @@ dependencies = ["utf8-width",]+[[package]]+name = "bytecheck"+version = "0.6.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "23cdc57ce23ac53c931e88a43d06d070a6fd142f2617be5855eb75efc9beb1c2"+dependencies = [+ "bytecheck_derive",+ "ptr_meta",+ "simdutf8",+]++[[package]]+name = "bytecheck_derive"+version = "0.6.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3db406d29fbcd95542e92559bed4d8ad92636d1ca8b3b72ede10b4bcc010e659"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 1.0.109",+]+[[package]]name = "bytecount"version = "0.6.7"@@ -859,6 +900,26 @@ version = "0.7.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "98cc8fbded0c607b7ba9dd60cd98df59af97e84d24e49c8557331cfc26d301ce"+[[package]]+name = "client-wasm"+version = "0.1.0"+dependencies = [+ "bincode",+ "js-sys",+ "optics",+ "serde",+ "serde-wasm-bindgen",+ "serde_json",+ "thiserror",+ "wasm-bindgen",+]++[[package]]+name = "cobs"+version = "0.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "67ba02a97a2bd10f4b59b25c7973101c79642302776489e030cd13cdab09ed15"+[[package]]name = "color_quant"version = "1.1.0"@@ -893,6 +954,26 @@ dependencies = ["windows-sys 0.52.0",]+[[package]]+name = "const_format"+version = "0.2.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e3a214c7af3d04997541b18d432afaff4c455e79e2029079647e72fc2bd27673"+dependencies = [+ "const_format_proc_macros",+]++[[package]]+name = "const_format_proc_macros"+version = "0.2.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c7f6ff08fd20f4f299298a28e2dfa8a8ba1036e6cd2460ac1de7b425d76f2500"+dependencies = [+ "proc-macro2",+ "quote",+ "unicode-xid",+]+[[package]]name = "constant_time_eq"version = "0.1.5"@@ -975,6 +1056,12 @@ dependencies = ["itertools 0.10.5",]+[[package]]+name = "critical-section"+version = "1.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7059fff8937831a9ae6f0fe4d658ffabf58f2ca96aa9dec1c889f936f705f216"+[[package]]name = "crossbeam-channel"version = "0.5.12"@@ -1337,6 +1424,12 @@ version = "1.10.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a"+[[package]]+name = "embedded-io"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ef1a6892d9eef45c8fa6b9e0086428a2cca8491aca8f787c534a3d6d0bcb3ced"+[[package]]name = "ena"version = "0.14.2"@@ -1928,6 +2021,15 @@ dependencies = ["serde",]+[[package]]+name = "hash32"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b0c35f58762feb77d74ebe43bdbc3210f09be9fe6742234d573bacc26ed92b67"+dependencies = [+ "byteorder 1.5.0",+]+[[package]]name = "hashbrown"version = "0.12.3"@@ -1948,6 +2050,20 @@ dependencies = ["serde",]+[[package]]+name = "heapless"+version = "0.7.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cdc6457c0eb62c71aac4bc17216026d8410337c4126773b9c5daba343f17964f"+dependencies = [+ "atomic-polyfill",+ "hash32",+ "rustc_version",+ "serde",+ "spin",+ "stable_deref_trait",+]+[[package]]name = "heck"version = "0.3.3"@@ -3109,6 +3225,7 @@ dependencies = [name = "optics"version = "0.1.0"dependencies = [+ "bincode","itertools 0.11.0","lalrpop","lalrpop-util",@@ -3433,6 +3550,31 @@ version = "1.6.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0"+[[package]]+name = "postcard"+version = "1.0.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a55c51ee6c0db07e68448e336cf8ea4131a620edefebf9893e759b2d793420f8"+dependencies = [+ "cobs",+ "const_format",+ "embedded-io",+ "heapless",+ "postcard-derive",+ "serde",+]++[[package]]+name = "postcard-derive"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc4b01218787dd4420daf63875163a787a78294ad48a24e9f6fa8c6507759a79"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 1.0.109",+]+[[package]]name = "powerfmt"version = "0.2.0"@@ -3547,6 +3689,26 @@ version = "2.0.11"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"+[[package]]+name = "ptr_meta"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0738ccf7ea06b608c10564b31debd4f5bc5e197fc8bfe088f68ae5ce81e7a4f1"+dependencies = [+ "ptr_meta_derive",+]++[[package]]+name = "ptr_meta_derive"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "16b845dbfca988fa33db069c0e230574d15a3088f147a87b64c7589eb662c9ac"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 1.0.109",+]+[[package]]name = "publicsuffix"version = "2.2.3"@@ -3843,6 +4005,15 @@ version = "0.8.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f"+[[package]]+name = "rend"+version = "0.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "71fe3824f5629716b1589be05dacd749f6aa084c87e00e016714a8cdfccc997c"+dependencies = [+ "bytecheck",+]+[[package]]name = "reqwest"version = "0.11.24"@@ -3923,6 +4094,35 @@ dependencies = ["rio_api",]+[[package]]+name = "rkyv"+version = "0.7.44"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5cba464629b3394fc4dbc6f940ff8f5b4ff5c7aef40f29166fd4ad12acbc99c0"+dependencies = [+ "bitvec",+ "bytecheck",+ "bytes",+ "hashbrown 0.12.3",+ "ptr_meta",+ "rend",+ "rkyv_derive",+ "seahash",+ "tinyvec",+ "uuid",+]++[[package]]+name = "rkyv_derive"+version = "0.7.44"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a7dddfff8de25e6f62b9d64e6e432bf1c6736c57d20323e15ee10435fbda7c65"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 1.0.109",+]+[[package]]name = "robotstxt-with-cache"version = "0.4.0"@@ -4225,6 +4425,12 @@ dependencies = ["syn 2.0.52",]+[[package]]+name = "seahash"+version = "4.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b"+[[package]]name = "security-framework"version = "2.9.2"@@ -4289,6 +4495,17 @@ dependencies = ["serde_derive",]+[[package]]+name = "serde-wasm-bindgen"+version = "0.6.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b"+dependencies = [+ "js-sys",+ "serde",+ "wasm-bindgen",+]+[[package]]name = "serde_derive"version = "1.0.197"@@ -4422,6 +4639,12 @@ version = "0.3.7"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"+[[package]]+name = "simdutf8"+version = "0.1.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a"+[[package]]name = "similar"version = "2.4.0"@@ -4523,6 +4746,7 @@ name = "stract"version = "0.1.0"dependencies = ["aes-gcm",+ "ahash 0.8.11","anyhow","async-channel","async-stream",@@ -4579,6 +4803,7 @@ dependencies = ["once_cell","openraft","optics",+ "postcard","proptest","proptest-derive","publicsuffix",@@ -4590,6 +4815,7 @@ dependencies = ["ring","rio_api","rio_turtle",+ "rkyv","robotstxt-with-cache","rocksdb","rust-s3",@@ -4618,6 +4844,7 @@ dependencies = ["utoipa-swagger-ui","uuid","whatlang",+ "xxhash-rust","zimba",]@@ -5494,6 +5721,12 @@ version = "0.9.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"+[[package]]+name = "unty"+version = "0.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a1a88342087869553c259588a3ec9ca73ce9b2d538b7051ba5789ff236b6c129"+[[package]]name = "url"version = "2.5.0"@@ -5614,6 +5847,12 @@ version = "0.9.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"+[[package]]+name = "virtue"+version = "0.0.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3b6826a786a78cf1bb0937507b5551fb6f827d66269a24b00af0de247b19bbc7"+[[package]]name = "wait-timeout"version = "0.2.0"@@ -5998,6 +6237,12 @@ dependencies = ["rustix",]+[[package]]+name = "xxhash-rust"+version = "0.8.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "927da81e25be1e1a2901d59b81b37dd2efd1fc9c9345a55007f09bf5a2d3ee03"+[[package]]name = "yaml-rust"version = "0.4.5"diff --git a/Cargo.toml b/Cargo.tomlindex bf1fc0f1..d89e8a18 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -1,5 +1,11 @@[workspace]-members = ["crates/core", "crates/optics", "crates/kuchiki", "crates/zimba"]+members = [+ "crates/core",+ "crates/optics",+ "crates/kuchiki",+ "crates/zimba",+ "crates/client-wasm",+]resolver = "2"[profile.release]@@ -10,6 +16,7 @@ lto = "thin"debug = true[workspace.dependencies]+ahash = "0.8.11"aes-gcm = "0.10.2"anyhow = { version = "1.0.72", features = ["backtrace"] }async-channel = "1.8.0"@@ -18,7 +25,9 @@ axum = "0.7.2"axum-extra = { version = "0.9.0" }axum-macros = "0.4.0"base64 = "0.21.4"-bincode = "1.3.3"+bincode = { git = "https://github.com/bincode-org/bincode", rev = "8f65c60", features = [+ "serde",+] }bindgen = "0.69.2"bitvec = "1.0.1"bytecount = "0.6.7"@@ -27,7 +36,11 @@ byteorder = "1.4.3"bzip2 = "0.4.4"candle-core = "0.3.3"candle-nn = "0.3.3"-openraft = { version = "0.9.1", features = ["storage-v2", "serde"] }+openraft = { version = "0.9.1", features = [+ "storage-v2",+ "serde",+ "tracing-log",+] }candle-transformers = "0.3.3"cc = { version = "1", features = ["parallel"] }chardetng = "0.1.17"@@ -71,6 +84,7 @@ min-max-heap = "1.3.0"nom = "7.1.3"num_cpus = "1.15.0"once_cell = "1.13.1"+postcard = { version = "1.0.8", features = ["experimental-derive", "alloc"] }proptest = "1.2.0"proptest-derive = "0.4.0"publicsuffix = "2.2.3"@@ -79,6 +93,7 @@ rand = "0.8.5"rayon = "1.5.3"regex = "1.6.0"reqwest = { version = "0.11.16", features = ["blocking", "stream", "json"] }+rkyv = { version = "0.7.44", features = ["validation", "bytecheck"] }ring = "0.17.3"rio_api = "0.8.4"rio_turtle = "0.8.4"@@ -116,6 +131,7 @@ uuid = "1.1.2"whatlang = { version = "0.16.0", features = ["serde"] }zstd = "0.13"tracing-test = "0.2.4"+xxhash-rust = { version = "0.8.10", features = ["xxh3", "const_xxh3"] }[profile.test.package]flate2.opt-level = 3diff --git a/assets/licenses.html b/assets/licenses.htmlindex cdb3b33a..43ff8b12 100644--- a/assets/licenses.html+++ b/assets/licenses.html@@ -44,13 +44,15 @@<h2>Overview of licenses:</h2><ul class="licenses-overview">- <li><a href="#Apache-2.0">Apache License 2.0</a> (395)</li>- <li><a href="#MIT">MIT License</a> (182)</li>+ <li><a href="#Apache-2.0">Apache License 2.0</a> (405)</li>+ <li><a href="#MIT">MIT License</a> (193)</li><li><a href="#MPL-2.0">Mozilla Public License 2.0</a> (9)</li><li><a href="#BSD-3-Clause">BSD 3-Clause "New" or "Revised" License</a> (8)</li><li><a href="#Unicode-DFS-2016">Unicode License Agreement - Data Files and Software (2016)</a> (5)</li><li><a href="#ISC">ISC License</a> (4)</li><li><a href="#CC0-1.0">Creative Commons Zero v1.0 Universal</a> (2)</li>+ <li><a href="#Zlib">zlib License</a> (2)</li>+ <li><a href="#BSL-1.0">Boost Software License 1.0</a> (1)</li><li><a href="#GPL-3.0">GNU General Public License v3.0 only</a> (1)</li><li><a href="#LGPL-3.0">GNU Lesser General Public License v3.0 only</a> (1)</li><li><a href="#OpenSSL">OpenSSL License</a> (1)</li>@@ -61,6 +63,190 @@<h2>All license text:</h2><ul class="licenses-list">+ <li class="license">+ <h3 id="Apache-2.0">Apache License 2.0</h3>+ <h4>Used by:</h4>+ <ul class="license-used-by">+ <li><a href=" https://github.com/rusticstuff/simdutf8 ">simdutf8 0.1.4</a></li>+ </ul>+ <pre class="license-text">+ Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for reasonable and customary use in describing the+ origin of the Work and reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may choose to offer,+ and charge a fee for, acceptance of support, warranty, indemnity,+ or other liability obligations and/or rights consistent with this+ License. However, in accepting such obligations, You may act only+ on Your own behalf and on Your sole responsibility, not on behalf+ of any other Contributor, and only if You agree to indemnify,+ defend, and hold each Contributor harmless for any liability+ incurred by, or claims asserted against, such Contributor by reason+ of your accepting any such warranty or additional liability.++ END OF TERMS AND CONDITIONS</pre>+ </li><li class="license"><h3 id="Apache-2.0">Apache License 2.0</h3><h4>Used by:</h4>@@ -3454,6 +3640,7 @@<li><a href=" https://github.com/rust-cli/anstyle.git ">anstyle-wincon 3.0.2</a></li><li><a href=" https://github.com/rust-cli/anstyle.git ">anstyle 1.0.6</a></li><li><a href=" https://github.com/clap-rs/clap ">clap 4.5.1</a></li>+ <li><a href=" https://github.com/jamesmunns/cobs.rs ">cobs 0.2.3</a></li><li><a href=" https://github.com/rust-cli/anstyle ">colorchoice 1.0.0</a></li><li><a href=" https://github.com/srijs/rust-crc32fast ">crc32fast 1.4.0</a></li><li><a href=" https://github.com/colin-kiegel/rust-derive-builder ">derive_builder 0.12.0</a></li>@@ -7023,6 +7210,7 @@ limitations under the License.</pre><li><a href=" https://github.com/nikomatsakis/ascii-canvas ">ascii-canvas 3.0.0</a></li><li><a href=" https://github.com/smol-rs/async-channel ">async-channel 1.9.0</a></li><li><a href=" https://github.com/Nullus157/async-compression ">async-compression 0.4.6</a></li>+ <li><a href=" https://github.com/embassy-rs/atomic-polyfill ">atomic-polyfill 1.0.3</a></li><li><a href=" https://github.com/cuviper/autocfg ">autocfg 1.1.0</a></li><li><a href=" https://github.com/rust-lang/backtrace-rs ">backtrace 0.3.69</a></li><li><a href=" https://github.com/marshallpierce/rust-base64 ">base64 0.13.1</a></li>@@ -7041,11 +7229,13 @@ limitations under the License.</pre><li><a href=" https://github.com/servo/core-foundation-rs ">core-foundation 0.9.4</a></li><li><a href=" https://github.com/bheisler/criterion.rs ">criterion-plot 0.5.0</a></li><li><a href=" https://github.com/bheisler/criterion.rs ">criterion 0.5.1</a></li>+ <li><a href=" https://github.com/rust-embedded/critical-section ">critical-section 1.1.2</a></li><li><a href=" https://github.com/crossbeam-rs/crossbeam ">crossbeam-channel 0.5.12</a></li><li><a href=" https://github.com/crossbeam-rs/crossbeam ">crossbeam-deque 0.8.5</a></li><li><a href=" https://github.com/crossbeam-rs/crossbeam ">crossbeam-epoch 0.9.18</a></li><li><a href=" https://github.com/crossbeam-rs/crossbeam ">crossbeam-utils 0.8.19</a></li><li><a href=" https://github.com/rayon-rs/either ">either 1.10.0</a></li>+ <li><a href=" https://github.com/embassy-rs/embedded-io ">embedded-io 0.4.0</a></li><li><a href=" https://github.com/rust-lang/ena ">ena 0.14.2</a></li><li><a href=" https://github.com/cuviper/equivalent ">equivalent 1.0.1</a></li><li><a href=" https://github.com/smol-rs/event-listener ">event-listener 2.5.3</a></li>@@ -7060,8 +7250,10 @@ limitations under the License.</pre><li><a href=" https://github.com/servo/futf ">futf 0.1.5</a></li><li><a href=" https://github.com/gimli-rs/gimli ">gimli 0.28.1</a></li><li><a href=" https://github.com/rust-lang/glob ">glob 0.3.1</a></li>+ <li><a href=" https://github.com/japaric/hash32 ">hash32 0.2.1</a></li><li><a href=" https://github.com/rust-lang/hashbrown ">hashbrown 0.12.3</a></li><li><a href=" https://github.com/rust-lang/hashbrown ">hashbrown 0.14.3</a></li>+ <li><a href=" https://github.com/japaric/heapless ">heapless 0.7.17</a></li><li><a href=" https://github.com/withoutboats/heck ">heck 0.3.3</a></li><li><a href=" https://github.com/withoutboats/heck ">heck 0.4.1</a></li><li><a href=" https://github.com/hermit-os/hermit-rs ">hermit-abi 0.3.9</a></li>@@ -7108,6 +7300,7 @@ limitations under the License.</pre><li><a href=" https://github.com/petgraph/petgraph ">petgraph 0.6.4</a></li><li><a href=" https://github.com/rust-lang/pkg-config-rs ">pkg-config 0.3.30</a></li><li><a href=" https://github.com/image-rs/image-png ">png 0.17.13</a></li>+ <li><a href=" https://github.com/jamesmunns/postcard ">postcard 1.0.8</a></li><li><a href=" https://github.com/dtolnay/proc-macro-hack ">proc-macro-hack 0.5.20+deprecated</a></li><li><a href=" https://github.com/proptest-rs/proptest ">proptest-derive 0.4.0</a></li><li><a href=" https://github.com/proptest-rs/proptest ">proptest 1.4.0</a></li>@@ -9904,6 +10097,7 @@ limitations under the License.<li><a href=" https://github.com/datafuselabs/openraft ">openraft-macros 0.9.1</a></li><li><a href=" https://github.com/datafuselabs/openraft ">openraft 0.9.1</a></li><li><a href=" https://github.com/sfackler/rust-openssl ">openssl 0.10.64</a></li>+ <li><a href=" https://github.com/jamesmunns/postcard ">postcard-derive 0.1.1</a></li><li><a href=" https://github.com/oxigraph/rio ">rio_api 0.8.4</a></li><li><a href=" https://github.com/oxigraph/rio ">rio_turtle 0.8.4</a></li><li><a href=" https://github.com/huggingface/safetensors ">safetensors 0.3.3</a></li>@@ -9915,6 +10109,7 @@ limitations under the License.<li><a href=" https://github.com/jedisct1/rust-siphash ">siphasher 0.3.11</a></li><li><a href=" https://github.com/mullvad/system-configuration-rs ">system-configuration-sys 0.5.0</a></li><li><a href=" https://github.com/mullvad/system-configuration-rs ">system-configuration 0.5.1</a></li>+ <li><a href=" https://crates.io/crates/unty ">unty 0.0.3</a></li><li><a href=" https://github.com/retep998/winapi-rs ">winapi-i686-pc-windows-gnu 0.4.0</a></li><li><a href=" https://github.com/retep998/winapi-rs ">winapi-x86_64-pc-windows-gnu 0.4.0</a></li><li><a href=" https://crates.io/crates/zune-inflate ">zune-inflate 0.2.54</a></li>@@ -10577,6 +10772,37 @@ SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVERCAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USEOF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.+</pre>+ </li>+ <li class="license">+ <h3 id="BSL-1.0">Boost Software License 1.0</h3>+ <h4>Used by:</h4>+ <ul class="license-used-by">+ <li><a href=" https://github.com/DoumanAsh/xxhash-rust ">xxhash-rust 0.8.10</a></li>+ </ul>+ <pre class="license-text">Boost Software License - Version 1.0 - August 17th, 2003++Permission is hereby granted, free of charge, to any person or organization+obtaining a copy of the software and accompanying documentation covered by+this license (the "Software") to use, reproduce, display, distribute,+execute, and transmit the Software, and to prepare derivative works of the+Software, and to permit third-parties to whom the Software is furnished to+do so, all subject to the following:++The copyright notices in the Software and this entire statement, including+the above license grant, this restriction and the following disclaimer,+must be included in all copies of the Software, in whole or in part, and+all derivative works of the Software, unless such copies or derivative+works are solely in the form of machine-executable object code generated by+a source language processor.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT+SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE+FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,+ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER+DEALINGS IN THE SOFTWARE.</pre></li><li class="license">@@ -12815,6 +13041,22 @@ FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS ORCOPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHERIN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR INCONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.</pre>+ </li>+ <li class="license">+ <h3 id="MIT">MIT License</h3>+ <h4>Used by:</h4>+ <ul class="license-used-by">+ <li><a href=" https://github.com/djkoloski/bytecheck ">bytecheck 0.6.12</a></li>+ <li><a href=" https://github.com/djkoloski/bytecheck ">bytecheck_derive 0.6.12</a></li>+ </ul>+ <pre class="license-text">Copyright 2020 David Koloski++Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.+</pre></li><li class="license"><h3 id="MIT">MIT License</h3>@@ -12830,6 +13072,25 @@ Permission is hereby granted, free of charge, to any person obtaining a copy ofThe above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.+</pre>+ </li>+ <li class="license">+ <h3 id="MIT">MIT License</h3>+ <h4>Used by:</h4>+ <ul class="license-used-by">+ <li><a href=" https://github.com/djkoloski/ptr_meta ">ptr_meta 0.1.4</a></li>+ <li><a href=" https://github.com/djkoloski/ptr_meta ">ptr_meta_derive 0.1.4</a></li>+ <li><a href=" https://github.com/djkoloski/rend ">rend 0.4.2</a></li>+ <li><a href=" https://github.com/rkyv/rkyv ">rkyv 0.7.44</a></li>+ <li><a href=" https://github.com/rkyv/rkyv ">rkyv_derive 0.7.44</a></li>+ </ul>+ <pre class="license-text">Copyright 2021 David Koloski++Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.</pre></li>@@ -13467,6 +13728,7 @@ SOFTWARE.<li><a href=" https://github.com/plotters-rs/plotters.git ">plotters-svg 0.3.5</a></li><li><a href=" https://github.com/plotters-rs/plotters ">plotters 0.3.5</a></li><li><a href=" https://github.com/durch/rust-s3 ">rust-s3 0.33.0</a></li>+ <li><a href=" https://gitlab.redox-os.org/redox-os/seahash ">seahash 4.1.0</a></li><li><a href=" https://github.com/tokio-rs/tokio ">tokio-macros 2.2.0</a></li><li><a href=" https://github.com/dbrgn/tracing-test ">tracing-test-macro 0.2.4</a></li><li><a href=" https://github.com/dbrgn/tracing-test ">tracing-test 0.2.4</a></li>@@ -13534,6 +13796,35 @@ furnished to do so, subject to the following conditions:The above copyright notice and this permission notice shall be included in allcopies or substantial portions of the Software.+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.+</pre>+ </li>+ <li class="license">+ <h3 id="MIT">MIT License</h3>+ <h4>Used by:</h4>+ <ul class="license-used-by">+ <li><a href=" https://github.com/RReverser/serde-wasm-bindgen ">serde-wasm-bindgen 0.6.5</a></li>+ </ul>+ <pre class="license-text">MIT License++Copyright (c) 2019 Cloudflare, Inc.++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ORIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE@@ -13688,6 +13979,36 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHERLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THESOFTWARE.</pre>+ </li>+ <li class="license">+ <h3 id="MIT">MIT License</h3>+ <h4>Used by:</h4>+ <ul class="license-used-by">+ <li><a href=" https://github.com/bincode-org/bincode ">bincode 2.0.0-rc.3</a></li>+ <li><a href=" https://github.com/bincode-org/bincode ">bincode_derive 2.0.0-rc.3</a></li>+ </ul>+ <pre class="license-text">The MIT License (MIT)++Copyright (c) 2014 Ty Overby++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.+</pre></li><li class="license"><h3 id="MIT">MIT License</h3>@@ -14182,11 +14503,11 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRES<h3 id="MIT">MIT License</h3><h4>Used by:</h4><ul class="license-used-by">- <li><a href=" https://github.com/servo/bincode ">bincode 1.3.3</a></li>+ <li><a href=" https://github.com/fizyk20/generic-array.git ">generic-array 0.14.7</a></li></ul><pre class="license-text">The MIT License (MIT)-Copyright (c) 2014 Ty Overby+Copyright (c) 2015 Bartłomiej KamińskiPermission is hereby granted, free of charge, to any person obtaining a copyof this software and associated documentation files (the "Software"), to deal@@ -14204,18 +14525,18 @@ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THEAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHERLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE-SOFTWARE.-</pre>+SOFTWARE.</pre></li><li class="license"><h3 id="MIT">MIT License</h3><h4>Used by:</h4><ul class="license-used-by">- <li><a href=" https://github.com/fizyk20/generic-array.git ">generic-array 0.14.7</a></li>+ <li><a href=" https://github.com/tafia/quick-xml ">quick-xml 0.26.0</a></li>+ <li><a href=" https://github.com/tafia/quick-xml ">quick-xml 0.30.0</a></li></ul><pre class="license-text">The MIT License (MIT)-Copyright (c) 2015 Bartłomiej Kamiński+Copyright (c) 2016 Johann TuffePermission is hereby granted, free of charge, to any person obtaining a copyof this software and associated documentation files (the "Software"), to deal@@ -14224,27 +14545,29 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sellcopies of the Software, and to permit persons to whom the Software isfurnished to do so, subject to the following conditions:-The above copyright notice and this permission notice shall be included in all-copies or substantial portions of the Software.++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ORIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THEAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHERLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE-SOFTWARE.</pre>+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.+</pre></li><li class="license"><h3 id="MIT">MIT License</h3><h4>Used by:</h4><ul class="license-used-by">- <li><a href=" https://github.com/tafia/quick-xml ">quick-xml 0.26.0</a></li>- <li><a href=" https://github.com/tafia/quick-xml ">quick-xml 0.30.0</a></li>+ <li><a href=" https://github.com/bincode-org/virtue ">virtue 0.0.16</a></li></ul><pre class="license-text">The MIT License (MIT)-Copyright (c) 2016 Johann Tuffe+Copyright (c) 2021 Victor KoendersPermission is hereby granted, free of charge, to any person obtaining a copyof this software and associated documentation files (the "Software"), to deal@@ -14253,18 +14576,16 @@ to use, copy, modify, merge, publish, distribute, sublicense, and/or sellcopies of the Software, and to permit persons to whom the Software isfurnished to do so, subject to the following conditions:--The above copyright notice and this permission notice shall be included in-all copies or substantial portions of the Software.-+The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ORIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THEAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHERLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN-THE SOFTWARE.+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.</pre></li><li class="license">@@ -16391,6 +16712,31 @@ TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION0. You just DO WHAT THE FUCK YOU WANT TO.</pre>+ </li>+ <li class="license">+ <h3 id="Zlib">zlib License</h3>+ <h4>Used by:</h4>+ <ul class="license-used-by">+ <li><a href=" https://github.com/rodrimati1992/const_format_crates/ ">const_format 0.2.32</a></li>+ <li><a href=" https://github.com/rodrimati1992/const_format_crates/ ">const_format_proc_macros 0.2.32</a></li>+ </ul>+ <pre class="license-text">Copyright (c) 2020 Matias Rodriguez.++This software is provided 'as-is', without any express or implied+warranty. In no event will the authors be held liable for any damages+arising from the use of this software.++Permission is granted to anyone to use this software for any purpose,+including commercial applications, and to alter it and redistribute it+freely, subject to the following restrictions:++1. The origin of this software must not be misrepresented; you must not+ claim that you wrote the original software. If you use this software+ in a product, an acknowledgment in the product documentation would be+ appreciated but is not required.+2. Altered source versions must be plainly marked as such, and must not be+ misrepresented as being the original software.+3. This notice may not be removed or altered from any source distribution.</pre></li><li class="license"><h3 id="zlib-acknowledgement">zlib/libpng License with Acknowledgement</h3>diff --git a/crates/client-wasm/Cargo.toml b/crates/client-wasm/Cargo.tomlindex e4e1c553..61ddc327 100644--- a/crates/client-wasm/Cargo.toml+++ b/crates/client-wasm/Cargo.toml@@ -1,8 +1,10 @@[package]+authors = ['Stract <[email protected]>']edition = "2021"+license = "AGPL-3.0"name = "client-wasm"+publish = falseversion = "0.1.0"-wasm-opt = false[package.metadata.wasm-pack.profile.release]wasm-opt = false@@ -18,12 +20,11 @@ crate-type = ["cdylib", "rlib"][dependencies]js-sys = "0.3.60"-serde = {version = "1.0.137", features = ["rc", "derive"]}+serde = { version = "1.0.137", features = ["rc", "derive"] }serde-wasm-bindgen = "0.6.3"serde_json = "1.0"thiserror = "1.0.31"wasm-bindgen = "0.2.83"+bincode = { workspace = true }-optics = {path = "../optics"}--[workspace]+optics = { path = "../optics" }diff --git a/crates/core/Cargo.toml b/crates/core/Cargo.tomlindex 9642695f..95dcfb99 100644--- a/crates/core/Cargo.toml+++ b/crates/core/Cargo.toml@@ -20,6 +20,7 @@ name = "stract"path = "src/main.rs"[dependencies]+ahash = { workspace = true }aes-gcm = { workspace = true }anyhow = { workspace = true }async-channel = { workspace = true }@@ -75,11 +76,13 @@ once_cell = { workspace = true }openraft = { workspace = true }optics = { path = "../optics" }publicsuffix = { workspace = true }+postcard = { workspace = true }quick-xml = { workspace = true }rand = { workspace = true }rayon = { workspace = true }regex = { workspace = true }reqwest = { workspace = true }+rkyv = { workspace = true }ring = { workspace = true }rio_api = { workspace = true }rio_turtle = { workspace = true }@@ -110,6 +113,7 @@ utoipa-swagger-ui = { workspace = true }uuid = { workspace = true }whatlang = { workspace = true }zimba = { path = "../zimba" }+xxhash-rust = { workspace = true }[target.'cfg(not(target_env = "msvc"))'.dependencies]tikv-jemallocator = { workspace = true }@@ -154,3 +158,6 @@ name = "bitvec_similarity"[[example]]name = "indexer"++[[example]]+name = "distributed_harmonic"diff --git a/crates/core/examples/distributed_harmonic.rs b/crates/core/examples/distributed_harmonic.rsnew file mode 100644index 00000000..4067ffa1--- /dev/null+++ b/crates/core/examples/distributed_harmonic.rs@@ -0,0 +1,84 @@+use std::time::Duration;++use clap::Parser;+use stract::distributed::member::ShardId;+use tracing_subscriber::util::SubscriberInitExt;++#[derive(Parser)]+struct Args {+ graph_path: String,+ output_path: String,+}++fn start_dht_thread() {+ std::thread::spawn(|| {+ let config = stract::entrypoint::ampc::dht::Config {+ node_id: 1,+ host: "0.0.0.0:3000".parse().unwrap(),+ shard: ShardId::new(1),+ seed_node: None,+ gossip: Some(stract::config::GossipConfig {+ cluster_id: "test".to_string(),+ seed_nodes: None,+ addr: "0.0.0.0:3001".parse().unwrap(),+ }),+ };++ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ .block_on(stract::entrypoint::ampc::dht::run(config))+ .unwrap();+ });+}++fn start_worker_thread(graph_path: String) {+ std::thread::spawn(|| {+ let config = stract::config::HarmonicWorkerConfig {+ gossip: stract::config::GossipConfig {+ cluster_id: "test".to_string(),+ seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),+ addr: "0.0.0.0:3003".parse().unwrap(),+ },+ shard: ShardId::new(1),+ graph_path,+ host: "0.0.0.0:3002".parse().unwrap(),+ };++ stract::entrypoint::ampc::harmonic_centrality::worker::run(config).unwrap();+ });+}++fn main() -> anyhow::Result<()> {+ tracing_subscriber::fmt()+ .with_max_level(tracing::Level::INFO)+ .without_time()+ .with_target(false)+ .finish()+ .init();++ tracing::info!("Starting distributed harmonic centrality worker");++ let args = Args::parse();+ start_dht_thread();+ std::thread::sleep(Duration::from_secs(1));+ start_worker_thread(args.graph_path);+ std::thread::sleep(Duration::from_secs(1));++ let config = stract::config::HarmonicCoordinatorConfig {+ gossip: stract::config::GossipConfig {+ cluster_id: "test".to_string(),+ seed_nodes: Some(vec!["0.0.0.0:3001".parse().unwrap()]),+ addr: "0.0.0.0:3005".parse().unwrap(),+ },+ host: "0.0.0.0:3004".parse().unwrap(),+ output_path: args.output_path,+ };++ let start = std::time::Instant::now();+ stract::entrypoint::ampc::harmonic_centrality::coordinator::run(config)?;+ tracing::info!("Calculated centrality in: {:?}", start.elapsed());++ Ok(())+}diff --git a/crates/core/src/ampc/coordinator.rs b/crates/core/src/ampc/coordinator.rsnew file mode 100644index 00000000..3880f382--- /dev/null+++ b/crates/core/src/ampc/coordinator.rs@@ -0,0 +1,218 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use rayon::prelude::*;+use std::{+ collections::{BTreeMap, VecDeque},+ time::Duration,+};++use super::{+ DhtConn, ExponentialBackoff, Finisher, Job, JobScheduled, RemoteWorker, Setup, Worker,+ WorkerRef,+};+use crate::Result;+use anyhow::anyhow;++pub struct Coordinator<J>+where+ J: Job,+{+ workers: BTreeMap<WorkerRef, <<J as Job>::Worker as Worker>::Remote>,+ setup: Box<dyn Setup<DhtTables = J::DhtTables>>,+ mappers: Vec<J::Mapper>,+}++impl<J> Coordinator<J>+where+ J: Job,+{+ pub fn new<S>(setup: S, workers: Vec<<<J as Job>::Worker as Worker>::Remote>) -> Self+ where+ S: Setup<DhtTables = J::DhtTables> + 'static,+ {+ Self {+ setup: Box::new(setup),+ mappers: Vec::new(),+ workers: workers+ .into_iter()+ .enumerate()+ .map(|(i, w)| (WorkerRef(i), w))+ .collect(),+ }+ }++ pub fn with_mapper(mut self, mapper: J::Mapper) -> Self {+ self.mappers.push(mapper);+ self+ }++ fn send_dht_to_workers(&self, dht: &DhtConn<J::DhtTables>) -> Result<()> {+ self.workers+ .par_iter()+ .map(|(_, worker)| worker.send_dht(dht))+ .collect::<Result<()>>()?;++ Ok(())+ }++ fn schedule_job(+ &self,+ job: J,+ mapper: J::Mapper,+ worker_jobs: &BTreeMap<WorkerRef, Result<Option<J>>>,+ ) -> Result<JobScheduled> {+ let potential_workers = worker_jobs+ .iter()+ .filter_map(|(r, j)| j.as_ref().ok().map(|_| *r))+ .filter(|r| job.is_schedulable(&self.workers[r]))+ .collect::<Vec<_>>();++ if potential_workers.is_empty() {+ return Err(anyhow!(+ "Failed to schedule job: no potential workers are responding."+ ));+ }++ // schedule remaining jobs to idle workers (if any)+ match potential_workers.iter().find(|r| {+ worker_jobs[r]+ .as_ref()+ .expect("references in `potential_workers` should only point to non-failed workers")+ .is_none()+ }) {+ Some(free_worker) => {+ self.workers[free_worker].schedule_job(&job, mapper)?;+ Ok(JobScheduled::Success(*free_worker))+ }+ None => Ok(JobScheduled::NoAvailableWorkers),+ }+ }++ fn await_scheduled_jobs(+ &self,+ mut scheduled_jobs: BTreeMap<WorkerRef, J>,+ mapper: J::Mapper,+ ) -> Result<()> {+ tracing::debug!("Awaiting scheduled jobs");+ let mut sleeper =+ ExponentialBackoff::new(Duration::from_millis(100), Duration::from_secs(10), 2.0);++ loop {+ let worker_jobs = self+ .workers+ .iter()+ .map(|(r, w)| (*r, w.current_job()))+ .collect::<BTreeMap<_, _>>();++ let mut finished_workers = 0;+ for (r, j) in worker_jobs.iter() {+ match j {+ Ok(Some(_)) => break,+ Ok(None) => finished_workers += 1,+ Err(_) => {+ if let Some(job) = &scheduled_jobs.remove(r) {+ match self.schedule_job(job.clone(), mapper.clone(), &worker_jobs)? {+ JobScheduled::Success(r) => {+ scheduled_jobs.insert(r, job.clone());+ break; // need to break to avoid double scheduling to same worker+ }+ JobScheduled::NoAvailableWorkers => {+ // retry scheduling.+ break;+ }+ }+ }+ }+ }+ }++ if finished_workers == self.workers.len() {+ break;+ }++ std::thread::sleep(sleeper.next());+ }++ Ok(())+ }++ pub fn run<F>(self, jobs: Vec<J>, finisher: F) -> Result<J::DhtTables>+ where+ F: Finisher<Job = J>,+ {+ let mut dht = self.setup.init_dht();+ dht.cleanup_prev_tables();++ self.setup.setup_first_round(dht.prev());+ self.setup.setup_first_round(dht.next());++ while !finisher.is_finished(dht.prev()) {+ tracing::debug!("Starting new round");+ self.setup.setup_round(dht.next());+ self.send_dht_to_workers(&dht)?;++ for mapper in &self.mappers {+ // run round+ let mut remaining_jobs: VecDeque<_> = jobs.clone().into_iter().collect();+ let mut sleeper = ExponentialBackoff::new(+ Duration::from_millis(100),+ Duration::from_secs(10),+ 2.0,+ );++ let mut scheduled_jobs: BTreeMap<WorkerRef, J> = BTreeMap::new();++ while let Some(job) = remaining_jobs.pop_front() {+ // get current status from workers+ let worker_jobs = self+ .workers+ .iter()+ .map(|(r, w)| (*r, w.current_job()))+ .collect::<BTreeMap<_, _>>();++ // handle failed workers+ for r in worker_jobs+ .iter()+ .filter_map(|(r, j)| j.as_ref().err().map(|_| r))+ {+ if let Some(job) = scheduled_jobs.remove(r) {+ remaining_jobs.push_front(job);+ }+ }++ match self.schedule_job(job.clone(), mapper.clone(), &worker_jobs)? {+ JobScheduled::Success(worker) => {+ scheduled_jobs.insert(worker, job);+ sleeper.success();+ }+ JobScheduled::NoAvailableWorkers => {+ remaining_jobs.push_front(job);+ let sleep_duration = sleeper.next();+ std::thread::sleep(sleep_duration);+ }+ }+ }++ self.await_scheduled_jobs(scheduled_jobs, mapper.clone())?;+ }++ dht.next_round();+ }++ Ok(dht.take_prev())+ }+}diff --git a/crates/core/src/ampc/dht/client.rs b/crates/core/src/ampc/dht/client.rsnew file mode 100644index 00000000..9649ced2--- /dev/null+++ b/crates/core/src/ampc/dht/client.rs@@ -0,0 +1,391 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use async_stream::stream;+use futures::Stream;+use rand::seq::SliceRandom;+use std::{+ collections::BTreeMap,+ net::SocketAddr,+ ops::{Bound, Range},+};++use crate::{distributed::member::ShardId, fast_stable_hash, Result};++use super::{+ network::api,+ store::{Key, Table, Value},+ upsert::UpsertEnum,+ UpsertAction,+};++#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct Node {+ api: api::RemoteClient,+}++impl Clone for Node {+ fn clone(&self) -> Self {+ Self {+ api: api::RemoteClient::new(self.api.addr()),+ }+ }+}++impl Node {+ pub fn new(addr: SocketAddr) -> Self {+ let api = api::RemoteClient::new(addr);++ Self { api }+ }++ pub async fn get(&self, table: Table, key: Key) -> Result<Option<Value>> {+ self.api.get(table, key).await+ }++ pub async fn batch_get(&self, table: Table, keys: Vec<Key>) -> Result<Vec<(Key, Value)>> {+ self.api.batch_get(table, keys).await+ }++ pub async fn set(&self, table: Table, key: Key, value: Value) -> Result<()> {+ self.api.set(table, key, value).await+ }++ pub async fn batch_set(&self, table: Table, values: Vec<(Key, Value)>) -> Result<()> {+ self.api.batch_set(table, values).await+ }++ pub async fn upsert<F: Into<UpsertEnum>>(+ &self,+ table: Table,+ upsert: F,+ key: Key,+ value: Value,+ ) -> Result<UpsertAction> {+ self.api.upsert(table, upsert, key, value).await+ }++ pub async fn batch_upsert<F: Into<UpsertEnum>>(+ &self,+ table: Table,+ upsert: F,+ values: Vec<(Key, Value)>,+ ) -> Result<Vec<(Key, UpsertAction)>> {+ let res = self.api.batch_upsert(table, upsert, values.clone()).await?;++ debug_assert_eq!(res.len(), values.len());+ debug_assert!(res+ .iter()+ .all(|(k, _)| values.iter().any(|(key, _)| key == k)));++ Ok(res)+ }++ pub async fn range_get(+ &self,+ table: Table,+ range: Range<Bound<Key>>,+ limit: Option<usize>,+ ) -> Result<Vec<(Key, Value)>> {+ self.api.range_get(table, range, limit).await+ }++ pub fn stream(&self, table: Table) -> impl Stream<Item = Result<(Key, Value)>> + '_ {+ const STREAM_BATCH_SIZE: usize = 1024;+ stream! {+ let mut prev_key = None;++ loop {+ let batch = self.range_get(+ table.clone(),+ prev_key+ .as_ref()+ .cloned()+ .map_or(Bound::Unbounded, Bound::Excluded)+ ..Bound::Unbounded,+ Some(STREAM_BATCH_SIZE),+ ).await?;++ if batch.is_empty() {+ break;+ }++ for (key, value) in batch {+ yield Ok((key.clone(), value));+ prev_key = Some(key);+ }+ }+ }+ }+}++#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct Shard {+ nodes: Vec<Node>,+}++impl Default for Shard {+ fn default() -> Self {+ Self::new()+ }+}++impl Shard {+ pub fn new() -> Self {+ Self { nodes: Vec::new() }+ }++ pub fn add_node(&mut self, addr: SocketAddr) {+ self.nodes.push(Node::new(addr));+ }++ pub fn node(&self) -> &Node {+ self.nodes.choose(&mut rand::thread_rng()).unwrap()+ }++ pub async fn get(&self, table: Table, key: Key) -> Result<Option<Value>> {+ self.node().get(table, key).await+ }++ pub async fn batch_get(&self, table: Table, keys: Vec<Key>) -> Result<Vec<(Key, Value)>> {+ self.node().batch_get(table, keys).await+ }++ pub async fn set(&self, table: Table, key: Key, value: Value) -> Result<()> {+ self.node().set(table, key, value).await+ }++ pub async fn batch_set(&self, table: Table, values: Vec<(Key, Value)>) -> Result<()> {+ self.node().batch_set(table, values).await+ }++ pub async fn upsert<F: Into<UpsertEnum>>(+ &self,+ table: Table,+ upsert: F,+ key: Key,+ value: Value,+ ) -> Result<UpsertAction> {+ self.node().upsert(table, upsert, key, value).await+ }++ pub async fn batch_upsert<F: Into<UpsertEnum>>(+ &self,+ table: Table,+ upsert: F,+ values: Vec<(Key, Value)>,+ ) -> Result<Vec<(Key, UpsertAction)>> {+ self.node().batch_upsert(table, upsert, values).await+ }++ pub fn stream(&self, table: Table) -> impl Stream<Item = Result<(Key, Value)>> + '_ {+ self.node().stream(table)+ }+}++#[derive(Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug)]+pub struct Client {+ ids: Vec<ShardId>,+ shards: BTreeMap<ShardId, Shard>,+}++impl Client {+ pub fn new(members: &[(ShardId, SocketAddr)]) -> Self {+ let mut shards = BTreeMap::new();++ for (shard, host) in members {+ shards+ .entry(*shard)+ .or_insert_with(Shard::new)+ .add_node(*host);+ }++ let ids = shards.keys().cloned().collect();++ Self { shards, ids }+ }++ pub fn shards(&self) -> &BTreeMap<ShardId, Shard> {+ &self.shards+ }++ pub fn add_node(&mut self, shard_id: ShardId, addr: SocketAddr) {+ self.shards.entry(shard_id).or_default().add_node(addr);++ self.ids = self.shards.keys().cloned().collect();+ }++ fn shard_id_for_key(&self, key: &[u8]) -> Result<&ShardId> {+ if self.ids.is_empty() {+ return Err(anyhow::anyhow!("No shards"));+ }++ let hash = fast_stable_hash(key);++ Ok(&self.ids[hash as usize % self.ids.len()])+ }++ fn shard_for_key(&self, key: &[u8]) -> Result<&Shard> {+ let shard_id = self.shard_id_for_key(key)?;+ Ok(self.shards.get(shard_id).unwrap())+ }++ pub async fn get(&self, table: Table, key: Key) -> Result<Option<Value>> {+ self.shard_for_key(key.as_bytes())?.get(table, key).await+ }++ pub async fn batch_get(&self, table: Table, keys: Vec<Key>) -> Result<Vec<(Key, Value)>> {+ let mut shard_keys: BTreeMap<ShardId, Vec<Key>> = BTreeMap::new();++ for key in keys {+ let shard = self.shard_id_for_key(key.as_bytes())?;+ shard_keys.entry(*shard).or_default().push(key);+ }++ let mut futures = Vec::with_capacity(shard_keys.len());++ for (shard_id, keys) in shard_keys {+ futures.push(self.shards[&shard_id].batch_get(table.clone(), keys));+ }++ Ok(futures::future::try_join_all(futures)+ .await?+ .into_iter()+ .flatten()+ .collect())+ }++ pub async fn set(&self, table: Table, key: Key, value: Value) -> Result<()> {+ self.shard_for_key(key.as_bytes())?+ .set(table, key, value)+ .await+ }++ pub async fn batch_set(&self, table: Table, values: Vec<(Key, Value)>) -> Result<()> {+ let mut shard_values: BTreeMap<ShardId, Vec<(Key, Value)>> = BTreeMap::new();++ for (key, value) in values {+ let shard = self.shard_id_for_key(key.as_bytes())?;+ shard_values.entry(*shard).or_default().push((key, value));+ }++ let mut futures = Vec::with_capacity(shard_values.len());++ for (shard_id, values) in shard_values {+ futures.push(self.shards[&shard_id].batch_set(table.clone(), values));+ }++ futures::future::try_join_all(futures).await?;++ Ok(())+ }++ pub async fn upsert<F: Into<UpsertEnum>>(+ &self,+ table: Table,+ upsert: F,+ key: Key,+ value: Value,+ ) -> Result<UpsertAction> {+ self.shard_for_key(key.as_bytes())?+ .upsert(table, upsert, key, value)+ .await+ }++ pub async fn batch_upsert<F: Into<UpsertEnum> + Clone>(+ &self,+ table: Table,+ upsert: F,+ values: Vec<(Key, Value)>,+ ) -> Result<Vec<(Key, UpsertAction)>> {+ let mut shard_values: BTreeMap<ShardId, Vec<(Key, Value)>> = BTreeMap::new();++ for (key, value) in values {+ let shard = self.shard_id_for_key(key.as_bytes())?;+ shard_values.entry(*shard).or_default().push((key, value));+ }++ let mut futures = Vec::with_capacity(shard_values.len());++ for (shard_id, values) in shard_values {+ futures.push(self.shards[&shard_id].batch_upsert(+ table.clone(),+ upsert.clone(),+ values,+ ));+ }++ Ok(futures::future::try_join_all(futures)+ .await?+ .into_iter()+ .flatten()+ .collect())+ }++ pub async fn drop_table(&self, table: Table) -> Result<()> {+ for shard in self.shards.values() {+ for node in &shard.nodes {+ node.api.drop_table(table.clone()).await?;+ }+ }++ Ok(())+ }++ pub async fn create_table(&self, table: Table) -> Result<()> {+ for shard in self.shards.values() {+ for node in &shard.nodes {+ node.api.create_table(table.clone()).await?;+ }+ }++ Ok(())+ }++ pub async fn all_tables(&self) -> Result<Vec<Table>> {+ let mut tables = Vec::new();++ for shard in self.shards.values() {+ for node in &shard.nodes {+ tables.extend(node.api.all_tables().await?);+ }+ }++ tables.sort();+ tables.dedup();++ Ok(tables)+ }++ pub async fn clone_table(&self, from: Table, to: Table) -> Result<()> {+ for shard in self.shards.values() {+ for node in &shard.nodes {+ node.api.clone_table(from.clone(), to.clone()).await?;+ }+ }++ Ok(())+ }++ pub fn stream(&self, table: Table) -> impl Stream<Item = Result<(Key, Value)>> + '_ {+ let mut streams = Vec::new();+ for shard in self.shards.values() {+ streams.push(Box::pin(shard.stream(table.clone())));+ }++ futures::stream::select_all(streams)+ }+}diff --git a/crates/core/src/mapreduce/dht/log_store.rs b/crates/core/src/ampc/dht/log_store.rssimilarity index 100%rename from crates/core/src/mapreduce/dht/log_store.rsrename to crates/core/src/ampc/dht/log_store.rsdiff --git a/crates/core/src/mapreduce/dht/mod.rs b/crates/core/src/ampc/dht/mod.rssimilarity index 61%rename from crates/core/src/mapreduce/dht/mod.rsrename to crates/core/src/ampc/dht/mod.rsindex 6667c906..1b9178f2 100644--- a/crates/core/src/mapreduce/dht/mod.rs+++ b/crates/core/src/ampc/dht/mod.rs@@ -28,26 +28,55 @@mod client;pub mod log_store;-mod network;+pub mod network;pub mod store;+pub mod upsert;-use network::api::{Get, Set};+use network::api::{+ AllTables, BatchSet, BatchUpsert, CloneTable, CreateTable, DropTable, Set, Upsert,+};use std::fmt::Debug;use std::io::Cursor;-use openraft::BasicNode;use openraft::TokioRuntime;-use self::network::Server;+pub use self::network::Server;-pub use network::api::RemoteClient as ApiClient;+pub use network::api::RemoteClient;pub use network::raft::RemoteClient as RaftClient;-pub use client::Client;+pub use crate::distributed::member::ShardId;+pub use client::{Client, Shard};+pub use store::{Key, Table, Value};+pub use upsert::*;pub type NodeId = u64;+#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Debug,+ Clone,+ PartialEq,+ Eq,+ Default,+)]+pub struct BasicNode {+ pub addr: String,+}++impl BasicNode {+ /// Creates as [`BasicNode`].+ pub fn new(addr: impl ToString) -> Self {+ Self {+ addr: addr.to_string(),+ }+ }+}+openraft::declare_raft_types!(/// Declare the type configuration for example K/V store.pub TypeConfig:@@ -60,17 +89,16 @@ openraft::declare_raft_types!(AsyncRuntime = TokioRuntime,);-#[macro_export]macro_rules! raft_sonic_request_response {($service:ident, [$($req:ident),*$(,)?]) => {- #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]+ #[derive(::serde::Serialize, ::serde::Deserialize, ::bincode::Decode, Clone, Debug)]pub enum Request {$($req($req),)*}- #[derive(serde::Serialize, serde::Deserialize, Clone, Debug)]+ #[derive(::serde::Serialize, ::serde::Deserialize, ::bincode::Encode, Clone, Debug)]pub enum Response {$($req(<$req as $crate::distributed::sonic::service::Message<$service>>::Response),@@ -78,18 +106,6 @@ macro_rules! raft_sonic_request_response {Empty,}- $(- impl TryFrom<Response> for <$req as $crate::distributed::sonic::service::Message<$service>>::Response {- type Error = $crate::distributed::sonic::Error;- fn try_from(res: Response) -> Result<Self, Self::Error> {- match res {- Response::$req(res) => Ok(res),- _ => Err($crate::distributed::sonic::Error::Application(anyhow::anyhow!("Invalid response for request from Raft"))),- }- }- }- )*-$(impl From<$req> for Request {fn from(req: $req) -> Self {@@ -97,29 +113,41 @@ macro_rules! raft_sonic_request_response {}})*- };+ }}-raft_sonic_request_response!(Server, [Get, Set]);+raft_sonic_request_response!(+ Server,+ [+ Set,+ BatchSet,+ Upsert,+ BatchUpsert,+ CreateTable,+ DropTable,+ AllTables,+ CloneTable+ ]+);#[cfg(test)]-mod tests {+pub mod tests {use std::{collections::BTreeMap, net::SocketAddr, sync::Arc};- use tests::network::api::RemoteClient;use tokio::sync::Mutex;use tracing_test::traced_test;- use crate::{distributed::sonic, free_socket_addr};+ use crate::{ampc::dht, distributed::sonic, free_socket_addr};use openraft::{error::InitializeError, Config};use proptest::prelude::*;use proptest_derive::Arbitrary;+ use futures::{pin_mut, TryStreamExt};use rand::seq::SliceRandom;use super::*;- async fn server(+ pub async fn server(id: u64,) -> anyhow::Result<(openraft::Raft<TypeConfig>,@@ -196,20 +224,30 @@ mod tests {let c1 = RemoteClient::new(addr1);let c2 = RemoteClient::new(addr2);- c1.set("hello".as_bytes().to_vec(), "world".as_bytes().to_vec())- .await?;+ let table = Table::from("test");- let res = c1.get("hello".as_bytes().to_vec()).await?;- assert_eq!(res, Some("world".as_bytes().to_vec()));+ c1.set(+ table.clone(),+ "hello".as_bytes().to_vec().into(),+ "world".as_bytes().to_vec().into(),+ )+ .await?;- let res = c2.get("hello".as_bytes().to_vec()).await?;- assert_eq!(res, Some("world".as_bytes().to_vec()));+ let res = c1.get(table.clone(), "hello".as_bytes().into()).await?;+ assert_eq!(res, Some("world".as_bytes().into()));- c2.set("hello".as_bytes().to_vec(), "world2".as_bytes().to_vec())- .await?;+ let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;+ assert_eq!(res, Some("world".as_bytes().into()));++ c2.set(+ table.clone(),+ "hello".as_bytes().into(),+ "world2".as_bytes().into(),+ )+ .await?;- let res = c1.get("hello".as_bytes().to_vec()).await?;- assert_eq!(res, Some("world2".as_bytes().to_vec()));+ let res = c1.get(table.clone(), "hello".as_bytes().into()).await?;+ assert_eq!(res, Some("world2".as_bytes().into()));Ok(())}@@ -218,8 +256,8 @@ mod tests {#[traced_test]async fn test_member_join() -> anyhow::Result<()> {let (raft1, server1, addr1) = server(1).await?;- let (raft2, server2, addr2) = server(2).await?;- let (raft3, server3, addr3) = server(3).await?;+ let (_, server2, addr2) = server(2).await?;+ let (_, server3, addr3) = server(3).await?;let servers = vec![server1, server2, server3];@@ -231,7 +269,7 @@ mod tests {});}- let members: BTreeMap<u64, _> = vec![(1, addr1), (2, addr2)]+ let members: BTreeMap<u64, _> = vec![(1, addr1)].into_iter().map(|(id, addr)| (id, BasicNode::new(addr))).collect();@@ -246,7 +284,58 @@ mod tests {}};- if let Err(e) = raft2.initialize(members.clone()).await {+ let rc1 = network::raft::RemoteClient::new(addr1).await?;++ rc1.join(2, addr2).await?;++ let c1 = RemoteClient::new(addr1);+ let c2 = RemoteClient::new(addr2);++ let table = Table::from("test");++ c1.set(+ table.clone(),+ "hello".as_bytes().into(),+ "world".as_bytes().into(),+ )+ .await?;+ tokio::time::sleep(std::time::Duration::from_secs(1)).await; // reads are not linearized (only eventual consistency)++ let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;++ assert_eq!(res, Some("world".as_bytes().into()));++ rc1.join(3, addr3).await?;+ tokio::time::sleep(std::time::Duration::from_secs(1)).await; // join is not blocking++ let c3 = RemoteClient::new(addr3);+ let res = c3.get(table.clone(), "hello".as_bytes().into()).await?;+ assert_eq!(res, Some("world".as_bytes().into()));++ Ok(())+ }++ #[tokio::test]+ #[traced_test]+ async fn test_stream() -> anyhow::Result<()> {+ let (raft, server, addr) = server(1).await?;++ let servers = vec![server];++ for server in servers {+ tokio::spawn(async move {+ loop {+ server.accept().await.unwrap();+ }+ });+ }++ let members: BTreeMap<u64, _> = vec![(1, addr)]+ .into_iter()+ .map(|(id, addr)| (id, BasicNode::new(addr)))+ .collect();++ if let Err(e) = raft.initialize(members.clone()).await {match e {openraft::error::RaftError::APIError(e) => match e {InitializeError::NotAllowed(_) => {}@@ -256,40 +345,96 @@ mod tests {}};- let c1 = RemoteClient::new(addr1);- let c2 = RemoteClient::new(addr2);+ let client = dht::client::Node::new(addr);+ let table = Table::from("test");- c1.set("hello".as_bytes().to_vec(), "world".as_bytes().to_vec())+ client+ .set(+ table.clone(),+ "hello".as_bytes().into(),+ "world".as_bytes().into(),+ ).await?;- let res = c2.get("hello".as_bytes().to_vec()).await?;+ client+ .set(+ table.clone(),+ "hello2".as_bytes().into(),+ "world2".as_bytes().into(),+ )+ .await?;- assert_eq!(res, Some("world".as_bytes().to_vec()));+ let stream = client.stream(table.clone());+ pin_mut!(stream);- let members: BTreeMap<u64, _> = vec![(1, addr1), (2, addr2), (3, addr3)]- .into_iter()- .map(|(id, addr)| (id, BasicNode::new(addr)))- .collect();+ let mut res = Vec::new();- raft3.initialize(members.clone()).await?;- let rc1 = network::raft::RemoteClient::new(1, BasicNode::new(addr1));- rc1.join(3, addr3, members.clone()).await?;+ while let Some((k, v)) = stream.try_next().await? {+ res.push((k, v));+ }- let c3 = RemoteClient::new(addr3);- let res = c3.get("hello".as_bytes().to_vec()).await?;- assert_eq!(res, Some("world".as_bytes().to_vec()));+ res.sort_by_key(|(k, _)| k.clone());++ assert_eq!(+ res,+ vec![+ ("hello".as_bytes().into(), "world".as_bytes().into()),+ ("hello2".as_bytes().into(), "world2".as_bytes().into()),+ ]+ );++ let table = Table::from("test2");++ const N: u64 = 100_000;+ client+ .batch_set(+ table.clone(),+ (0..N)+ .into_iter()+ .map(|i| {+ (+ i.to_be_bytes().as_ref().into(),+ i.to_be_bytes().as_ref().into(),+ )+ })+ .collect::<Vec<(Key, Value)>>(),+ )+ .await?;++ let stream = client.stream(table.clone());+ pin_mut!(stream);++ let mut res = Vec::new();++ while let Some((k, v)) = stream.try_next().await? {+ res.push((k, v));+ }++ res.sort_by_key(|(k, _)| k.clone());+ assert_eq!(res.len(), N as usize);++ for i in 0..N as usize {+ assert_eq!(+ res[i],+ (+ i.to_be_bytes().as_ref().into(),+ i.to_be_bytes().as_ref().into(),+ )+ );+ }Ok(())}#[tokio::test]#[traced_test]+ #[ignore = "comitted logs must be stored in stable storage for raft to be able to recover from a node crash"]+ // see: https://docs.rs/openraft/latest/openraft/docs/faq/index.html#what-will-happen-when-data-gets-lostasync fn test_node_crash() -> anyhow::Result<()> {let (raft1, server1, addr1) = server(1).await?;let (raft2, server2, addr2) = server(2).await?;- let (raft3, server3, addr3) = server(3).await?;- let servers = vec![server1, server2, server3];+ let servers = vec![server1, server2];let mut handles = Vec::new();for server in servers {@@ -300,7 +445,7 @@ mod tests {}));}- let members: BTreeMap<u64, _> = vec![(1, addr1), (2, addr2), (3, addr3)]+ let members: BTreeMap<u64, _> = vec![(1, addr1)].into_iter().map(|(id, addr)| (id, BasicNode::new(addr))).collect();@@ -315,41 +460,25 @@ mod tests {}};- if let Err(e) = raft2.initialize(members.clone()).await {- match e {- openraft::error::RaftError::APIError(e) => match e {- InitializeError::NotAllowed(_) => {}- InitializeError::NotInMembers(_) => panic!("{:?}", e),- },- openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),- }- };-- if let Err(e) = raft3.initialize(members.clone()).await {- match e {- openraft::error::RaftError::APIError(e) => match e {- InitializeError::NotAllowed(_) => {}- InitializeError::NotInMembers(_) => panic!("{:?}", e),- },- openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),- }- };+ let rc1 = network::raft::RemoteClient::new(addr1).await?;+ rc1.join(2, addr2).await?;let c1 = RemoteClient::new(addr1);- let c2 = RemoteClient::new(addr2);-- let rc1 = network::raft::RemoteClient::new(1, BasicNode::new(addr1));- c1.set("hello".as_bytes().to_vec(), "world".as_bytes().to_vec())- .await?;+ let table = Table::from("test");- let res = c1.get("hello".as_bytes().to_vec()).await?;- assert_eq!(res, Some("world".as_bytes().to_vec()));+ c1.set(+ table.clone(),+ "hello".as_bytes().into(),+ "world".as_bytes().into(),+ )+ .await?;- let res = c2.get("hello".as_bytes().to_vec()).await?;- assert_eq!(res, Some("world".as_bytes().to_vec()));+ let res = c1.get(table.clone(), "hello".as_bytes().into()).await?;+ assert_eq!(res, Some("world".as_bytes().into()));// crash node 2+ tracing::info!("crashing node 2");handles[1].abort();drop(raft2);@@ -360,38 +489,53 @@ mod tests {}});- rc1.join(2, addr2, members.clone()).await?;+ rc1.join(2, addr2).await?;+ tokio::time::sleep(std::time::Duration::from_secs(1)).await;let c2 = RemoteClient::new(addr2);- let res = c2.get("hello".as_bytes().to_vec()).await?;- assert_eq!(res, Some("world".as_bytes().to_vec()));+ let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;+ assert_eq!(res, Some("world".as_bytes().into()));// crash node 2 again+ tracing::info!("crashing node 2 again");handles[1].abort();drop(raft2);- c1.set("hello".as_bytes().to_vec(), "world2".as_bytes().to_vec())- .await?;+ c1.set(+ table.clone(),+ "hello".as_bytes().into(),+ "world2".as_bytes().into(),+ )+ .await?;- let (raft2, server2, addr2) = server(2).await?;+ let (_raft2, server2, addr2) = server(2).await?;handles[1] = tokio::spawn(async move {loop {server2.accept().await.unwrap();}});- raft2.initialize(members.clone()).await?;- rc1.join(2, addr2, members.clone()).await?;+ rc1.join(2, addr2).await?;+ tokio::time::sleep(std::time::Duration::from_secs(1)).await;let c2 = RemoteClient::new(addr2);- let res = c2.get("hello".as_bytes().to_vec()).await?;- assert_eq!(res, Some("world2".as_bytes().to_vec()));+ let res = c2.get(table.clone(), "hello".as_bytes().into()).await?;+ assert_eq!(res, Some("world2".as_bytes().into()));Ok(())}- #[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Arbitrary)]+ #[derive(+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Arbitrary,+ )]enum Action {Set { key: Vec<u8>, value: Vec<u8> },// get actions[prev_key % actions.len()]@@ -457,34 +601,35 @@ mod tests {let clients = Arc::new(vec![c1, c2]);let shared_actions = Arc::new(actions.clone());+ let table = Table::from("test");for (i, action) in actions.into_iter().enumerate() {match action {Action::Set { key, value } => {let client = clients.choose(&mut rand::thread_rng()).unwrap();- client.set(key.clone(), value.clone()).await.unwrap();+ client.set(table.clone(), key.clone().into(), value.clone().into()).await.unwrap();ground_truth.lock().await.insert(key.clone(), value.clone());}Action::Get { prev_key } => {let client = clients.choose(&mut rand::thread_rng()).unwrap();- client.set(b"ensure-linearized-read".to_vec(), vec![]).await.unwrap();+ client.set(table.clone(), b"ensure-linearized-read".to_vec().into(), vec![].into()).await.unwrap();let key = if i == 0 {b"non-existent-key".to_vec()} else {- match shared_actions[dbg!(prev_key % dbg!(i))] {+ match shared_actions[prev_key % i] {Action::Set { ref key, .. } => {- dbg!(key.clone())+ key.clone()},Action::Get { .. } => b"non-existent-key".to_vec(),}};- let res = client.get(key.clone()).await.unwrap();+ let res = client.get(table.clone(), key.clone().into()).await.unwrap();let expected = ground_truth.lock().await.get(&key).cloned();- assert_eq!(res, expected);+ assert_eq!(res.map(|v| v.as_bytes().to_vec()), expected);}}}diff --git a/crates/core/src/ampc/dht/network/api.rs b/crates/core/src/ampc/dht/network/api.rsnew file mode 100644index 00000000..73430344--- /dev/null+++ b/crates/core/src/ampc/dht/network/api.rs@@ -0,0 +1,999 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use std::{+ net::SocketAddr,+ ops::{Bound, Range},+ sync::Arc,+ time::Duration,+};++use crate::{+ ampc::dht::{+ store::{Key, Table, Value},+ upsert::UpsertEnum,+ BasicNode, UpsertAction,+ },+ distributed::retry_strategy::RandomBackoff,+ Result,+};+use anyhow::anyhow;+use openraft::error::{ClientWriteError, ForwardToLeader, RaftError};+use tokio::sync::RwLock;++use crate::{ampc::dht::NodeId, distributed::sonic};++use super::Server;++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct Set {+ pub table: Table,+ pub key: Key,+ pub value: Value,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct BatchSet {+ pub table: Table,+ pub values: Arc<Vec<(Key, Value)>>,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct Upsert {+ pub table: Table,+ pub key: Key,+ pub value: Value,+ pub upsert_fn: UpsertEnum,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct BatchUpsert {+ pub table: Table,+ pub values: Arc<Vec<(Key, Value)>>,+ pub upsert_fn: UpsertEnum,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct Get {+ pub table: Table,+ pub key: Key,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct BatchGet {+ pub table: Table,+ pub keys: Arc<Vec<Key>>,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct DropTable {+ pub table: Table,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct CreateTable {+ pub table: Table,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct AllTables;++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct CloneTable {+ pub from: Table,+ pub to: Table,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct RangeGet {+ pub table: Table,+ pub range: Range<Bound<Key>>,+ pub limit: Option<usize>,+}++impl sonic::service::Message<Server> for Set {+ type Response = Result<+ (),+ crate::bincode_utils::SerdeCompat<RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>,+ >;++ async fn handle(self, server: &Server) -> Self::Response {+ tracing::debug!("received set request: {:?}", self);++ match server.raft.client_write(self.into()).await {+ Ok(_) => Ok(()),+ Err(e) => Err(crate::bincode_utils::SerdeCompat(e)),+ }+ }+}++impl sonic::service::Message<Server> for BatchSet {+ type Response = Result<+ (),+ crate::bincode_utils::SerdeCompat<RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>,+ >;++ async fn handle(self, server: &Server) -> Self::Response {+ tracing::debug!("received batch set request: {:?}", self);++ match server.raft.client_write(self.into()).await {+ Ok(_) => Ok(()),+ Err(e) => Err(crate::bincode_utils::SerdeCompat(e)),+ }+ }+}++impl sonic::service::Message<Server> for Upsert {+ type Response = Result<+ UpsertAction,+ crate::bincode_utils::SerdeCompat<RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>,+ >;++ async fn handle(self, server: &Server) -> Self::Response {+ tracing::debug!("received upsert request: {:?}", self);++ match server.raft.client_write(self.into()).await {+ Ok(res) => match res.data {+ crate::ampc::dht::Response::Upsert(res) => res,+ _ => panic!("unexpected response from raft"),+ },+ Err(e) => Err(crate::bincode_utils::SerdeCompat(e)),+ }+ }+}++impl sonic::service::Message<Server> for BatchUpsert {+ type Response = Result<+ Vec<(Key, UpsertAction)>,+ crate::bincode_utils::SerdeCompat<RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>,+ >;++ async fn handle(self, server: &Server) -> Self::Response {+ tracing::debug!("received batch upsert request: {:?}", self);++ match server.raft.client_write(self.into()).await {+ Ok(res) => match res.data {+ crate::ampc::dht::Response::BatchUpsert(res) => res,+ _ => panic!("unexpected response from raft"),+ },+ Err(e) => Err(crate::bincode_utils::SerdeCompat(e)),+ }+ }+}++impl sonic::service::Message<Server> for Get {+ type Response = Option<Value>;++ async fn handle(self, server: &Server) -> Self::Response {+ server+ .state_machine_store+ .state_machine+ .read()+ .await+ .db+ .get(&self.table, &self.key)+ }+}++impl sonic::service::Message<Server> for BatchGet {+ type Response = Vec<(Key, Value)>;++ async fn handle(self, server: &Server) -> Self::Response {+ server+ .state_machine_store+ .state_machine+ .read()+ .await+ .db+ .batch_get(&self.table, &self.keys)+ }+}++impl sonic::service::Message<Server> for DropTable {+ type Response = Result<+ (),+ crate::bincode_utils::SerdeCompat<RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>,+ >;++ async fn handle(self, server: &Server) -> Self::Response {+ match server.raft.client_write(self.into()).await {+ Ok(_) => Ok(()),+ Err(e) => Err(crate::bincode_utils::SerdeCompat(e)),+ }+ }+}++impl sonic::service::Message<Server> for CreateTable {+ type Response = Result<+ (),+ crate::bincode_utils::SerdeCompat<RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>,+ >;++ async fn handle(self, server: &Server) -> Self::Response {+ match server.raft.client_write(self.into()).await {+ Ok(_) => Ok(()),+ Err(e) => Err(crate::bincode_utils::SerdeCompat(e)),+ }+ }+}++impl sonic::service::Message<Server> for AllTables {+ type Response = Result<+ Vec<Table>,+ crate::bincode_utils::SerdeCompat<RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>,+ >;++ async fn handle(self, server: &Server) -> Self::Response {+ match server.raft.client_write(self.into()).await {+ Ok(res) => match res.data {+ crate::ampc::dht::Response::AllTables(tables) => tables,+ _ => panic!("unexpected response from raft"),+ },+ Err(e) => Err(crate::bincode_utils::SerdeCompat(e)),+ }+ }+}++impl sonic::service::Message<Server> for CloneTable {+ type Response = Result<+ (),+ crate::bincode_utils::SerdeCompat<RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>,+ >;++ async fn handle(self, server: &Server) -> Self::Response {+ match server.raft.client_write(self.into()).await {+ Ok(_) => Ok(()),+ Err(e) => Err(crate::bincode_utils::SerdeCompat(e)),+ }+ }+}++impl sonic::service::Message<Server> for RangeGet {+ type Response = Vec<(Key, Value)>;++ async fn handle(self, server: &Server) -> Self::Response {+ server+ .state_machine_store+ .state_machine+ .read()+ .await+ .db+ .range_get(&self.table, self.range, self.limit)+ }+}++impl std::fmt::Debug for RemoteClient {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.debug_struct("RemoteClient")+ .field("self_remote", &self.self_remote.addr())+ .finish()+ }+}++#[derive(serde::Serialize, serde::Deserialize)]+pub struct RemoteClient {+ self_remote: sonic::replication::RemoteClient<Server>,+ #[serde(skip)]+ likely_leader: RwLock<Option<sonic::replication::RemoteClient<Server>>>,+}++impl bincode::Encode for RemoteClient {+ fn encode<E: bincode::enc::Encoder>(+ &self,+ encoder: &mut E,+ ) -> Result<(), bincode::error::EncodeError> {+ self.self_remote.encode(encoder)+ }+}++impl bincode::Decode for RemoteClient {+ fn decode<D: bincode::de::Decoder>(+ decoder: &mut D,+ ) -> Result<Self, bincode::error::DecodeError> {+ let self_remote = sonic::replication::RemoteClient::decode(decoder)?;+ Ok(Self {+ self_remote,+ likely_leader: RwLock::new(None),+ })+ }+}++impl<'de> bincode::BorrowDecode<'de> for RemoteClient {+ fn borrow_decode<D: bincode::de::BorrowDecoder<'de>>(+ decoder: &mut D,+ ) -> Result<Self, bincode::error::DecodeError> {+ let self_remote = sonic::replication::RemoteClient::borrow_decode(decoder)?;+ Ok(Self {+ self_remote,+ likely_leader: RwLock::new(None),+ })+ }+}++impl RemoteClient {+ pub fn new(addr: SocketAddr) -> Self {+ Self {+ self_remote: sonic::replication::RemoteClient::new(addr),+ likely_leader: RwLock::new(None),+ }+ }++ pub fn addr(&self) -> SocketAddr {+ self.self_remote.addr()+ }++ fn retry_strat() -> impl Iterator<Item = std::time::Duration> {+ RandomBackoff::new(+ std::time::Duration::from_millis(200),+ std::time::Duration::from_secs(1),+ )+ }++ pub async fn set(&self, table: Table, key: Key, value: Value) -> Result<()> {+ for backoff in Self::retry_strat() {+ let res = self+ .likely_leader+ .read()+ .await+ .as_ref()+ .unwrap_or(&self.self_remote)+ .send_with_timeout(+ &Set {+ table: table.clone(),+ key: key.clone(),+ value: value.clone(),+ },+ Duration::from_secs(5),+ )+ .await;++ match res {+ Ok(res) => match res {+ Ok(_) => return Ok(()),+ Err(crate::bincode_utils::SerdeCompat(RaftError::APIError(e))) => match e {+ ClientWriteError::ForwardToLeader(ForwardToLeader {+ leader_id: _,+ leader_node,+ }) => match leader_node {+ Some(leader_node) => {+ let mut likely_leader = self.likely_leader.write().await;+ *likely_leader = Some(sonic::replication::RemoteClient::new(+ leader_node+ .addr+ .parse()+ .expect("node addr should always be valid addr"),+ ));+ }+ None => {+ tokio::time::sleep(backoff).await;+ }+ },+ ClientWriteError::ChangeMembershipError(_) => {+ unreachable!(".set() should not change membership")+ }+ },+ Err(crate::bincode_utils::SerdeCompat(RaftError::Fatal(e))) => {+ return Err(e.into())+ }+ },+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to set key"))+ }++ pub async fn batch_set(&self, table: Table, values: Vec<(Key, Value)>) -> Result<()> {+ let values = Arc::new(values);++ for backoff in Self::retry_strat() {+ let res = self+ .likely_leader+ .read()+ .await+ .as_ref()+ .unwrap_or(&self.self_remote)+ .send_with_timeout(+ &BatchSet {+ table: table.clone(),+ values: values.clone(),+ },+ Duration::from_secs(5),+ )+ .await;++ match res {+ Ok(res) => match res {+ Ok(_) => return Ok(()),+ Err(crate::bincode_utils::SerdeCompat(RaftError::APIError(e))) => match e {+ ClientWriteError::ForwardToLeader(ForwardToLeader {+ leader_id: _,+ leader_node,+ }) => match leader_node {+ Some(leader_node) => {+ let mut likely_leader = self.likely_leader.write().await;+ *likely_leader = Some(sonic::replication::RemoteClient::new(+ leader_node+ .addr+ .parse()+ .expect("node addr should always be valid addr"),+ ));+ }+ None => {+ tokio::time::sleep(backoff).await;+ }+ },+ ClientWriteError::ChangeMembershipError(_) => {+ unreachable!(".batch_set() should not change membership")+ }+ },+ Err(crate::bincode_utils::SerdeCompat(RaftError::Fatal(e))) => {+ return Err(e.into())+ }+ },+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to batch set values"))+ }++ pub async fn get(&self, table: Table, key: Key) -> Result<Option<Value>> {+ for backoff in Self::retry_strat() {+ match self+ .self_remote+ .send_with_timeout(+ &Get {+ table: table.clone(),+ key: key.clone(),+ },+ Duration::from_secs(5),+ )+ .await+ {+ Ok(res) => return Ok(res),+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to get key"))+ }++ pub async fn batch_get(&self, table: Table, keys: Vec<Key>) -> Result<Vec<(Key, Value)>> {+ let keys = Arc::new(keys);++ for backoff in Self::retry_strat() {+ match self+ .self_remote+ .send_with_timeout(+ &BatchGet {+ table: table.clone(),+ keys: keys.clone(),+ },+ Duration::from_secs(5),+ )+ .await+ {+ Ok(res) => return Ok(res),+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to batch get keys"))+ }++ pub async fn drop_table(&self, table: Table) -> Result<()> {+ for backoff in Self::retry_strat() {+ let res = self+ .likely_leader+ .read()+ .await+ .as_ref()+ .unwrap_or(&self.self_remote)+ .send_with_timeout(+ &DropTable {+ table: table.clone(),+ },+ Duration::from_secs(5),+ )+ .await;++ match res {+ Ok(res) => match res {+ Ok(_) => return Ok(()),+ Err(crate::bincode_utils::SerdeCompat(RaftError::APIError(e))) => match e {+ ClientWriteError::ForwardToLeader(ForwardToLeader {+ leader_id: _,+ leader_node,+ }) => match leader_node {+ Some(leader_node) => {+ let mut likely_leader = self.likely_leader.write().await;+ *likely_leader = Some(sonic::replication::RemoteClient::new(+ leader_node+ .addr+ .parse()+ .expect("node addr should always be valid addr"),+ ));+ }+ None => {+ tokio::time::sleep(backoff).await;+ }+ },+ ClientWriteError::ChangeMembershipError(_) => {+ unreachable!(".drop_table() should not change membership")+ }+ },+ Err(crate::bincode_utils::SerdeCompat(RaftError::Fatal(e))) => {+ return Err(e.into())+ }+ },+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to drop table"))+ }++ pub async fn create_table(&self, table: Table) -> Result<()> {+ for backoff in Self::retry_strat() {+ let res = self+ .likely_leader+ .read()+ .await+ .as_ref()+ .unwrap_or(&self.self_remote)+ .send_with_timeout(+ &CreateTable {+ table: table.clone(),+ },+ Duration::from_secs(5),+ )+ .await;++ match res {+ Ok(res) => match res {+ Ok(_) => return Ok(()),+ Err(crate::bincode_utils::SerdeCompat(RaftError::APIError(e))) => match e {+ ClientWriteError::ForwardToLeader(ForwardToLeader {+ leader_id: _,+ leader_node,+ }) => match leader_node {+ Some(leader_node) => {+ let mut likely_leader = self.likely_leader.write().await;+ *likely_leader = Some(sonic::replication::RemoteClient::new(+ leader_node+ .addr+ .parse()+ .expect("node addr should always be valid addr"),+ ));+ }+ None => {+ tokio::time::sleep(backoff).await;+ }+ },+ ClientWriteError::ChangeMembershipError(_) => {+ unreachable!(".create_table() should not change membership")+ }+ },+ Err(crate::bincode_utils::SerdeCompat(RaftError::Fatal(e))) => {+ return Err(e.into())+ }+ },+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to create table"))+ }++ pub async fn all_tables(&self) -> Result<Vec<Table>> {+ for backoff in Self::retry_strat() {+ let res = self+ .likely_leader+ .read()+ .await+ .as_ref()+ .unwrap_or(&self.self_remote)+ .send_with_timeout(&AllTables, Duration::from_secs(5))+ .await;++ match res {+ Ok(res) => match res {+ Ok(res) => return Ok(res),+ Err(crate::bincode_utils::SerdeCompat(RaftError::APIError(e))) => match e {+ ClientWriteError::ForwardToLeader(ForwardToLeader {+ leader_id: _,+ leader_node,+ }) => match leader_node {+ Some(leader_node) => {+ let mut likely_leader = self.likely_leader.write().await;+ *likely_leader = Some(sonic::replication::RemoteClient::new(+ leader_node+ .addr+ .parse()+ .expect("node addr should always be valid addr"),+ ));+ }+ None => {+ tokio::time::sleep(backoff).await;+ }+ },+ ClientWriteError::ChangeMembershipError(_) => {+ unreachable!(".all_tables() should not change membership")+ }+ },+ Err(crate::bincode_utils::SerdeCompat(RaftError::Fatal(e))) => {+ return Err(e.into())+ }+ },+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to get tables"))+ }++ pub async fn clone_table(&self, from: Table, to: Table) -> Result<()> {+ for backoff in Self::retry_strat() {+ let res = self+ .likely_leader+ .read()+ .await+ .as_ref()+ .unwrap_or(&self.self_remote)+ .send_with_timeout(+ &CloneTable {+ from: from.clone(),+ to: to.clone(),+ },+ Duration::from_secs(5),+ )+ .await;++ match res {+ Ok(res) => match res {+ Ok(res) => return Ok(res),+ Err(crate::bincode_utils::SerdeCompat(RaftError::APIError(e))) => match e {+ ClientWriteError::ForwardToLeader(ForwardToLeader {+ leader_id: _,+ leader_node,+ }) => match leader_node {+ Some(leader_node) => {+ let mut likely_leader = self.likely_leader.write().await;+ *likely_leader = Some(sonic::replication::RemoteClient::new(+ leader_node+ .addr+ .parse()+ .expect("node addr should always be valid addr"),+ ));+ }+ None => {+ tokio::time::sleep(backoff).await;+ }+ },+ ClientWriteError::ChangeMembershipError(_) => {+ unreachable!(".clone_table() should not change membership")+ }+ },+ Err(crate::bincode_utils::SerdeCompat(RaftError::Fatal(e))) => {+ return Err(e.into())+ }+ },+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to clone table"))+ }++ pub async fn upsert<F: Into<UpsertEnum>>(+ &self,+ table: Table,+ upsert: F,+ key: Key,+ value: Value,+ ) -> Result<UpsertAction> {+ let upsert = upsert.into();++ for backoff in Self::retry_strat() {+ let res = self+ .likely_leader+ .read()+ .await+ .as_ref()+ .unwrap_or(&self.self_remote)+ .send_with_timeout(+ &Upsert {+ table: table.clone(),+ key: key.clone(),+ value: value.clone(),+ upsert_fn: upsert.clone(),+ },+ Duration::from_secs(5),+ )+ .await;++ match res {+ Ok(res) => match res {+ Ok(res) => return Ok(res),+ Err(crate::bincode_utils::SerdeCompat(RaftError::APIError(e))) => match e {+ ClientWriteError::ForwardToLeader(ForwardToLeader {+ leader_id: _,+ leader_node,+ }) => match leader_node {+ Some(leader_node) => {+ let mut likely_leader = self.likely_leader.write().await;+ *likely_leader = Some(sonic::replication::RemoteClient::new(+ leader_node+ .addr+ .parse()+ .expect("node addr should always be valid addr"),+ ));+ }+ None => {+ tokio::time::sleep(backoff).await;+ }+ },+ ClientWriteError::ChangeMembershipError(_) => {+ unreachable!(".upert() should not change membership")+ }+ },+ Err(crate::bincode_utils::SerdeCompat(RaftError::Fatal(e))) => {+ return Err(e.into())+ }+ },+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to perform upsert"))+ }++ pub async fn batch_upsert<F: Into<UpsertEnum>>(+ &self,+ table: Table,+ upsert: F,+ values: Vec<(Key, Value)>,+ ) -> Result<Vec<(Key, UpsertAction)>> {+ let upsert = upsert.into();+ let values = Arc::new(values);++ for backoff in Self::retry_strat() {+ let res = self+ .likely_leader+ .read()+ .await+ .as_ref()+ .unwrap_or(&self.self_remote)+ .send_with_timeout(+ &BatchUpsert {+ table: table.clone(),+ upsert_fn: upsert.clone(),+ values: values.clone(),+ },+ Duration::from_secs(5),+ )+ .await;++ match res {+ Ok(res) => match res {+ Ok(res) => return Ok(res),+ Err(crate::bincode_utils::SerdeCompat(RaftError::APIError(e))) => match e {+ ClientWriteError::ForwardToLeader(ForwardToLeader {+ leader_id: _,+ leader_node,+ }) => match leader_node {+ Some(leader_node) => {+ let mut likely_leader = self.likely_leader.write().await;+ *likely_leader = Some(sonic::replication::RemoteClient::new(+ leader_node+ .addr+ .parse()+ .expect("node addr should always be valid addr"),+ ));+ }+ None => {+ tokio::time::sleep(backoff).await;+ }+ },+ ClientWriteError::ChangeMembershipError(_) => {+ unreachable!(".batch_upsert() should not change membership")+ }+ },+ Err(crate::bincode_utils::SerdeCompat(RaftError::Fatal(e))) => {+ return Err(e.into())+ }+ },+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to batch upsert values"))+ }++ pub async fn range_get(+ &self,+ table: Table,+ range: Range<Bound<Key>>,+ limit: Option<usize>,+ ) -> Result<Vec<(Key, Value)>> {+ for backoff in Self::retry_strat() {+ let res = self+ .self_remote+ .send_with_timeout(+ &RangeGet {+ table: table.clone(),+ range: range.clone(),+ limit,+ },+ Duration::from_secs(5),+ )+ .await;++ tracing::debug!(".range_get() got response: {res:?}");++ match res {+ Ok(res) => return Ok(res),+ Err(e) => match e {+ sonic::Error::IO(_)+ | sonic::Error::ConnectionTimeout+ | sonic::Error::RequestTimeout+ | sonic::Error::PoolCreation => {+ tokio::time::sleep(backoff).await;+ }+ sonic::Error::BadRequest+ | sonic::Error::BodyTooLarge {+ body_size: _,+ max_size: _,+ }+ | sonic::Error::Application(_) => return Err(e.into()),+ },+ }+ }++ Err(anyhow!("failed to perform range get"))+ }+}diff --git a/crates/core/src/ampc/dht/network/mod.rs b/crates/core/src/ampc/dht/network/mod.rsnew file mode 100644index 00000000..95a89f55--- /dev/null+++ b/crates/core/src/ampc/dht/network/mod.rs@@ -0,0 +1,114 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++pub mod api;+pub mod raft;++use api::{+ AllTables, BatchGet, BatchSet, BatchUpsert, CloneTable, CreateTable, DropTable, Get, RangeGet,+ Set, Upsert,+};+use std::{collections::BTreeMap, net::SocketAddr, sync::Arc};++use openraft::{Raft, RaftNetworkFactory};++use self::raft::RemoteClient;+use crate::distributed::sonic::service::sonic_service;++use super::{store::StateMachineStore, BasicNode, NodeId, TypeConfig};++#[derive(Clone)]+pub struct Network;++impl RaftNetworkFactory<TypeConfig> for Network {+ type Network = RemoteClient;++ async fn new_client(&mut self, _: NodeId, node: &BasicNode) -> Self::Network {+ RemoteClient::new(node.addr.parse().unwrap()).await.unwrap()+ }+}++#[derive(bincode::Encode, bincode::Decode, serde::Serialize, serde::Deserialize, Debug, Clone)]+pub struct AppendEntries(#[bincode(with_serde)] openraft::raft::AppendEntriesRequest<TypeConfig>);+#[derive(bincode::Encode, bincode::Decode, serde::Serialize, serde::Deserialize, Debug)]+pub struct AppendEntriesResponse(+ #[bincode(with_serde)] pub openraft::raft::AppendEntriesResponse<NodeId>,+);++#[derive(bincode::Encode, bincode::Decode, serde::Serialize, serde::Deserialize, Debug, Clone)]+pub struct InstallSnapshot(+ #[bincode(with_serde)] pub openraft::raft::InstallSnapshotRequest<TypeConfig>,+);+#[derive(bincode::Encode, bincode::Decode, serde::Serialize, serde::Deserialize, Debug)]+pub struct InstallSnapshotResponse(+ #[bincode(with_serde)] pub openraft::raft::InstallSnapshotResponse<NodeId>,+);++#[derive(bincode::Encode, bincode::Decode, serde::Serialize, serde::Deserialize, Debug, Clone)]+pub struct Vote(#[bincode(with_serde)] pub openraft::raft::VoteRequest<NodeId>);+#[derive(bincode::Encode, bincode::Decode, serde::Serialize, serde::Deserialize, Debug, Clone)]+pub struct VoteResponse(#[bincode(with_serde)] pub openraft::raft::VoteResponse<NodeId>);++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct AddLearner {+ pub id: NodeId,+ pub addr: SocketAddr,+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct AddNodes {+ members: BTreeMap<NodeId, BasicNode>,+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct Metrics;++sonic_service!(+ Server,+ [+ AppendEntries,+ InstallSnapshot,+ Vote,+ Metrics,+ AddLearner,+ AddNodes,+ Get,+ BatchGet,+ Set,+ BatchSet,+ Upsert,+ BatchUpsert,+ DropTable,+ CreateTable,+ AllTables,+ CloneTable,+ RangeGet,+ ]+);++pub struct Server {+ raft: Raft<TypeConfig>,+ state_machine_store: Arc<StateMachineStore>,+}++impl Server {+ pub fn new(raft: Raft<TypeConfig>, state_machine_store: Arc<StateMachineStore>) -> Self {+ Self {+ raft,+ state_machine_store,+ }+ }+}diff --git a/crates/core/src/mapreduce/dht/network/raft.rs b/crates/core/src/ampc/dht/network/raft.rssimilarity index 56%rename from crates/core/src/mapreduce/dht/network/raft.rsrename to crates/core/src/ampc/dht/network/raft.rsindex 0e1297f4..3e1a13d2 100644--- a/crates/core/src/mapreduce/dht/network/raft.rs+++ b/crates/core/src/ampc/dht/network/raft.rs@@ -23,53 +23,81 @@ use std::{use openraft::{error::{ClientWriteError, ForwardToLeader, InstallSnapshotError, RaftError},network::RPCOption,- BasicNode, ChangeMembers, RaftNetwork,+ ChangeMembers, RaftMetrics, RaftNetwork,};use tokio::sync::RwLock;use crate::{+ ampc::{+ self,+ dht::{BasicNode, NodeId, TypeConfig},+ },distributed::{retry_strategy::ExponentialBackoff,sonic::{self, service::Connection},},- mapreduce::dht::{NodeId, TypeConfig},Result,};use super::{AddLearner, AddNodes, AppendEntries, AppendEntriesResponse, InstallSnapshot,- InstallSnapshotResponse, Server, Vote, VoteResponse,+ InstallSnapshotResponse, Metrics, Server, Vote, VoteResponse,};impl sonic::service::Message<Server> for AppendEntries {- type Response = Result<AppendEntriesResponse, RaftError<NodeId>>;+ type Response =+ Result<AppendEntriesResponse, crate::bincode_utils::SerdeCompat<RaftError<NodeId>>>;async fn handle(self, server: &Server) -> Self::Response {tracing::debug!("received append entries request: {:?}", self);- server.raft.append_entries(self).await+ Ok(AppendEntriesResponse(+ server+ .raft+ .append_entries(self.0)+ .await+ .map_err(crate::bincode_utils::SerdeCompat)?,+ ))}}impl sonic::service::Message<Server> for InstallSnapshot {- type Response = Result<InstallSnapshotResponse, RaftError<NodeId, InstallSnapshotError>>;+ type Response = Result<+ InstallSnapshotResponse,+ crate::bincode_utils::SerdeCompat<RaftError<NodeId, InstallSnapshotError>>,+ >;async fn handle(self, server: &Server) -> Self::Response {tracing::debug!("received install snapshot request: {:?}", self);- server.raft.install_snapshot(self).await+ Ok(InstallSnapshotResponse(+ server+ .raft+ .install_snapshot(self.0)+ .await+ .map_err(crate::bincode_utils::SerdeCompat)?,+ ))}}impl sonic::service::Message<Server> for Vote {- type Response = Result<VoteResponse, RaftError<NodeId>>;+ type Response = Result<VoteResponse, crate::bincode_utils::SerdeCompat<RaftError<NodeId>>>;async fn handle(self, server: &Server) -> Self::Response {tracing::debug!("received vote request: {:?}", self);- server.raft.vote(self).await+ Ok(VoteResponse(+ server+ .raft+ .vote(self.0)+ .await+ .map_err(crate::bincode_utils::SerdeCompat)?,+ ))}}impl sonic::service::Message<Server> for AddLearner {- type Response = Result<(), RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>;+ type Response = Result<+ (),+ crate::bincode_utils::SerdeCompat<RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>,+ >;async fn handle(self, server: &Server) -> Self::Response {tracing::debug!("received add learner request: {:?}", self);@@ -79,29 +107,50 @@ impl sonic::service::Message<Server> for AddLearner {server.raft- .change_membership(ChangeMembers::RemoveVoters(rem.clone()), true)- .await?;+ .change_membership(ChangeMembers::RemoveVoters(rem.clone()), false)+ .await+ .map_err(crate::bincode_utils::SerdeCompat)?;server.raft- .change_membership(ChangeMembers::RemoveNodes(rem.clone()), true)- .await?;+ .change_membership(ChangeMembers::RemoveNodes(rem.clone()), false)+ .await+ .map_err(crate::bincode_utils::SerdeCompat)?;let node = BasicNode::new(self.addr);- server.raft.add_learner(self.id, node, false).await?;+ server+ .raft+ .add_learner(self.id, node, false)+ .await+ .map_err(crate::bincode_utils::SerdeCompat)?;Ok(())}}+impl sonic::service::Message<Server> for Metrics {+ type Response = crate::bincode_utils::SerdeCompat<RaftMetrics<NodeId, BasicNode>>;++ async fn handle(self, server: &Server) -> Self::Response {+ tracing::debug!("received metrics request: {:?}", self);+ let metrics = server.raft.metrics().borrow().clone();++ crate::bincode_utils::SerdeCompat(metrics)+ }+}+impl sonic::service::Message<Server> for AddNodes {- type Response = Result<(), RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>;+ type Response = Result<+ (),+ crate::bincode_utils::SerdeCompat<RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>,+ >;async fn handle(self, server: &Server) -> Self::Response {tracing::debug!("received add nodes request: {:?}", self);server.raft- .change_membership(ChangeMembers::AddNodes(self.members), true)- .await?;+ .change_membership(ChangeMembers::AddNodes(self.members), false)+ .await+ .map_err(crate::bincode_utils::SerdeCompat)?;Ok(())}@@ -110,6 +159,28 @@ impl sonic::service::Message<Server> for AddNodes {type RPCError<E = openraft::error::Infallible> =openraft::error::RPCError<NodeId, BasicNode, RaftError<NodeId, E>>;+async fn metrics(+ client: &sonic::replication::RemoteClient<Server>,+) -> Result<RaftMetrics<NodeId, BasicNode>> {+ let rpc = Metrics;+ let retry = ExponentialBackoff::from_millis(500)+ .with_limit(Duration::from_secs(60))+ .take(5);++ for backoff in retry {+ let res = client+ .send_with_timeout(&rpc, Duration::from_secs(30))+ .await;++ match res {+ Ok(crate::bincode_utils::SerdeCompat(res)) => return Ok(res),+ Err(_) => tokio::time::sleep(backoff).await,+ };+ }++ Err(anyhow::anyhow!("failed to get metrics"))+}+pub struct RemoteClient {target: NodeId,node: BasicNode,@@ -118,30 +189,33 @@ pub struct RemoteClient {}impl RemoteClient {- pub fn new(target: NodeId, node: BasicNode) -> Self {- let addr: SocketAddr = node.addr.parse().expect("addr is not a valid address");+ pub async fn new(addr: SocketAddr) -> Result<Self> {let inner = sonic::replication::RemoteClient::new(addr);let likely_leader = RwLock::new(inner.clone());+ let metrics = metrics(&inner).await?;- Self {- target,- node,+ Ok(Self {+ target: metrics.id,+ node: BasicNode::new(addr),inner,likely_leader,- }+ })}- async fn raft_conn<E: std::error::Error>(&self) -> Result<Connection<Server>, RPCError<E>> {- self.inner- .conn()- .await- .map_err(|e| RPCError::Unreachable(openraft::error::Unreachable::new(&e)))+ async fn raft_conn<E: std::error::Error>(+ &self,+ ) -> Result<Connection<Server>, crate::bincode_utils::SerdeCompat<RPCError<E>>> {+ self.inner.conn().await.map_err(|e| {+ crate::bincode_utils::SerdeCompat(RPCError::Unreachable(+ openraft::error::Unreachable::new(&e),+ ))+ })}async fn send_raft_rpc<R, E>(&self,rpc: R,option: RPCOption,- ) -> Result<R::Response, RPCError<E>>+ ) -> Result<R::Response, crate::bincode_utils::SerdeCompat<RPCError<E>>>whereR: sonic::service::Wrapper<Server>,E: std::error::Error,@@ -151,7 +225,9 @@ impl RemoteClient {.await.map_err(|e| match e {sonic::Error::ConnectionTimeout | sonic::Error::RequestTimeout => {- RPCError::Unreachable(openraft::error::Unreachable::new(&e))+ crate::bincode_utils::SerdeCompat(RPCError::Unreachable(+ openraft::error::Unreachable::new(&e),+ ))}_ => {panic!("unexpected error: {:?}", e)@@ -159,6 +235,10 @@ impl RemoteClient {})}+ pub async fn metrics(&self) -> Result<RaftMetrics<NodeId, BasicNode>> {+ metrics(&self.inner).await+ }+async fn add_learner(&self, id: NodeId, addr: SocketAddr) -> Result<()> {let rpc = AddLearner { id, addr };let retry = ExponentialBackoff::from_millis(500)@@ -176,7 +256,7 @@ impl RemoteClient {match res {Ok(res) => match res {Ok(_) => return Ok(()),- Err(RaftError::APIError(e)) => match e {+ Err(crate::bincode_utils::SerdeCompat(RaftError::APIError(e))) => match e {ClientWriteError::ForwardToLeader(ForwardToLeader {leader_id: _,leader_node,@@ -196,11 +276,12 @@ impl RemoteClient {tokio::time::sleep(backoff).await}},- Err(RaftError::Fatal(e)) => return Err(e.into()),+ Err(crate::bincode_utils::SerdeCompat(RaftError::Fatal(e))) => {+ return Err(e.into())+ }},Err(e) => match e {sonic::Error::IO(_)- | sonic::Error::Serialization(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout| sonic::Error::PoolCreation => {@@ -234,7 +315,7 @@ impl RemoteClient {match res {Ok(res) => match res {Ok(_) => return Ok(()),- Err(RaftError::APIError(e)) => match e {+ Err(crate::bincode_utils::SerdeCompat(RaftError::APIError(e))) => match e {ClientWriteError::ForwardToLeader(ForwardToLeader {leader_id: _,leader_node,@@ -254,11 +335,12 @@ impl RemoteClient {tokio::time::sleep(backoff).await}},- Err(RaftError::Fatal(e)) => return Err(e.into()),+ Err(crate::bincode_utils::SerdeCompat(RaftError::Fatal(e))) => {+ return Err(e.into())+ }},Err(e) => match e {sonic::Error::IO(_)- | sonic::Error::Serialization(_)| sonic::Error::ConnectionTimeout| sonic::Error::RequestTimeout| sonic::Error::PoolCreation => {@@ -277,14 +359,22 @@ impl RemoteClient {unreachable!("should continue to retry");}- pub async fn join(- &self,- id: NodeId,- addr: SocketAddr,- new_all_nodes: BTreeMap<NodeId, BasicNode>,- ) -> Result<()> {+ pub async fn join(&self, id: NodeId, addr: SocketAddr) -> Result<()> {self.add_learner(id, addr).await?;- self.add_nodes(new_all_nodes).await?;+ let metrics = self.metrics().await?;++ let nodes_in_cluster = metrics+ .membership_config+ .nodes()+ .map(|(nid, node)| (*nid, node.clone()))+ .collect::<BTreeMap<_, _>>();++ debug_assert!(+ nodes_in_cluster.contains_key(&id),+ "node should be in the cluster"+ );++ self.add_nodes(nodes_in_cluster).await?;Ok(())}@@ -293,42 +383,64 @@ impl RemoteClient {impl RaftNetwork<TypeConfig> for RemoteClient {async fn append_entries(&mut self,- rpc: AppendEntries,+ rpc: openraft::raft::AppendEntriesRequest<ampc::dht::TypeConfig>,option: RPCOption,- ) -> Result<AppendEntriesResponse, RPCError> {- self.send_raft_rpc(rpc, option).await?.map_err(|e| {- openraft::error::RemoteError {- target: self.target,- target_node: Some(self.node.clone()),- source: e,- }- .into()- })+ ) -> Result<openraft::raft::AppendEntriesResponse<NodeId>, RPCError> {+ Ok(self+ .send_raft_rpc(AppendEntries(rpc), option)+ .await+ .map_err(|crate::bincode_utils::SerdeCompat(e)| e)?+ .map_err(|crate::bincode_utils::SerdeCompat(e)| -> RPCError {+ openraft::error::RemoteError {+ target: self.target,+ target_node: Some(self.node.clone()),+ source: e,+ }+ .into()+ })?+ .0)}async fn install_snapshot(&mut self,- rpc: InstallSnapshot,+ rpc: openraft::raft::InstallSnapshotRequest<ampc::dht::TypeConfig>,option: RPCOption,- ) -> Result<InstallSnapshotResponse, RPCError<InstallSnapshotError>> {- self.send_raft_rpc(rpc, option).await?.map_err(|e| {- openraft::error::RemoteError {- target: self.target,- target_node: Some(self.node.clone()),- source: e,- }- .into()- })+ ) -> Result<openraft::raft::InstallSnapshotResponse<NodeId>, RPCError<InstallSnapshotError>>+ {+ Ok(self+ .send_raft_rpc(InstallSnapshot(rpc), option)+ .await+ .map_err(|crate::bincode_utils::SerdeCompat(e)| e)?+ .map_err(+ |crate::bincode_utils::SerdeCompat(e)| -> RPCError<InstallSnapshotError> {+ openraft::error::RemoteError {+ target: self.target,+ target_node: Some(self.node.clone()),+ source: e,+ }+ .into()+ },+ )?+ .0)}- async fn vote(&mut self, rpc: Vote, option: RPCOption) -> Result<VoteResponse, RPCError> {- self.send_raft_rpc(rpc, option).await?.map_err(|e| {- openraft::error::RemoteError {- target: self.target,- target_node: Some(self.node.clone()),- source: e,- }- .into()- })+ async fn vote(+ &mut self,+ rpc: openraft::raft::VoteRequest<NodeId>,+ option: RPCOption,+ ) -> Result<openraft::raft::VoteResponse<NodeId>, RPCError> {+ Ok(self+ .send_raft_rpc(Vote(rpc), option)+ .await+ .map_err(|crate::bincode_utils::SerdeCompat(e)| e)?+ .map_err(|crate::bincode_utils::SerdeCompat(e)| -> RPCError {+ openraft::error::RemoteError {+ target: self.target,+ target_node: Some(self.node.clone()),+ source: e,+ }+ .into()+ })?+ .0)}}diff --git a/crates/core/src/ampc/dht/store.rs b/crates/core/src/ampc/dht/store.rsnew file mode 100644index 00000000..b6c0973e--- /dev/null+++ b/crates/core/src/ampc/dht/store.rs@@ -0,0 +1,571 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>+use std::collections::BTreeMap;+use std::fmt::Debug;+use std::io::Cursor;+use std::ops::Bound;+use std::ops::Range;+use std::sync::Arc;+use std::sync::Mutex;++use openraft::storage::RaftStateMachine;+use openraft::storage::Snapshot;+use openraft::Entry;+use openraft::EntryPayload;+use openraft::LogId;+use openraft::RaftSnapshotBuilder;+use openraft::RaftTypeConfig;+use openraft::SnapshotMeta;+use openraft::StorageError;+use openraft::StorageIOError;+use openraft::StoredMembership;+use tokio::sync::RwLock;++use crate::ampc::dht::network::api;++use super::upsert::UpsertEnum;+use super::upsert::UpsertFn;+use super::BasicNode;+use super::NodeId;+use super::TypeConfig;+use super::UpsertAction;+use super::{Request, Response};++#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Debug,+ Clone,+ PartialOrd,+ Ord,+ PartialEq,+ Eq,+)]+#[serde(transparent)]+pub struct Table(String);++impl Table {+ pub fn as_str(&self) -> &str {+ &self.0+ }+}++impl From<String> for Table {+ fn from(v: String) -> Self {+ Self(v)+ }+}++impl From<&str> for Table {+ fn from(v: &str) -> Self {+ Self(v.to_string())+ }+}++#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Debug,+ Clone,+ PartialOrd,+ Ord,+ PartialEq,+ Eq,+)]+#[serde(transparent)]+pub struct Key(Vec<u8>);++impl Key {+ pub fn as_bytes(&self) -> &[u8] {+ &self.0+ }+}++impl From<Vec<u8>> for Key {+ fn from(v: Vec<u8>) -> Self {+ Self(v)+ }+}++impl From<&[u8]> for Key {+ fn from(v: &[u8]) -> Self {+ Self(v.to_vec())+ }+}++#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Debug,+ Clone,+ PartialEq,+ Eq,+)]+#[serde(transparent)]+pub struct Value(Vec<u8>);++impl Value {+ pub fn as_bytes(&self) -> &[u8] {+ &self.0+ }++ pub fn as_bytes_mut(&mut self) -> &mut [u8] {+ &mut self.0+ }+}++impl From<Vec<u8>> for Value {+ fn from(v: Vec<u8>) -> Self {+ Self(v)+ }+}++impl From<&[u8]> for Value {+ fn from(v: &[u8]) -> Self {+ Self(v.to_vec())+ }+}++#[derive(+ serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Default, Clone,+)]+pub struct Db {+ data: BTreeMap<Table, BTreeMap<Arc<Key>, Arc<Value>>>,+}++impl Db {+ pub fn drop_table(&mut self, table: &Table) {+ let table = self.data.remove(table);+ if let Some(table) = table {+ // drop in background as some tables can be large+ std::thread::spawn(move || {+ drop(table);+ });+ }+ }++ pub fn get(&self, table: &Table, key: &Key) -> Option<Value> {+ self.data+ .get(table)+ .and_then(|m| m.get(key).map(|v| v.as_ref().clone()))+ }++ pub fn set(&mut self, table: Table, key: Key, value: Value) {+ self.data+ .entry(table)+ .or_default()+ .insert(Arc::new(key), Arc::new(value));+ }++ pub fn batch_set(&mut self, table: Table, values: Vec<(Key, Value)>) {+ let table = self.data.entry(table).or_default();++ // for some reason, the entry API seems to be faster than using extend or inserts+ for (k, v) in values {+ match table.entry(Arc::new(k)) {+ std::collections::btree_map::Entry::Occupied(mut e) => {+ *e.get_mut() = Arc::new(v);+ }+ std::collections::btree_map::Entry::Vacant(e) => {+ e.insert(Arc::new(v));+ }+ }+ }+ }++ pub fn upsert(+ &mut self,+ table: Table,+ upsert_fn: &UpsertEnum,+ key: Key,+ value: Value,+ ) -> UpsertAction {+ let table = self.data.entry(table).or_default();++ match table.get_mut(&key) {+ Some(old) => {+ let merged = upsert_fn.upsert(old.as_ref().clone(), value);++ let has_changed = merged != **old;++ *old = Arc::new(merged);++ if has_changed {+ UpsertAction::Merged+ } else {+ UpsertAction::NoChange+ }+ }+ None => {+ table.insert(Arc::new(key), Arc::new(value));+ UpsertAction::Inserted+ }+ }+ }++ pub fn batch_upsert(+ &mut self,+ table: Table,+ upsert_fn: &UpsertEnum,+ values: Vec<(Key, Value)>,+ ) -> Vec<(Key, UpsertAction)> {+ let table = self.data.entry(table).or_default();+ let mut res = Vec::with_capacity(values.len());++ for (key, value) in values {+ match table.get_mut(&key) {+ Some(old) => {+ let merged = upsert_fn.upsert(old.as_ref().clone(), value);+ let has_changed = merged != **old;++ *old = Arc::new(merged);++ if has_changed {+ res.push((key, UpsertAction::Merged));+ } else {+ res.push((key, UpsertAction::NoChange));+ }+ }+ None => {+ table.insert(Arc::new(key.clone()), Arc::new(value));+ res.push((key, UpsertAction::Inserted));+ }+ }+ }++ res+ }++ pub fn clone_table(&mut self, from: &Table, to: Table) {+ let data = self.data.get(from).cloned().unwrap_or_default();+ self.data.insert(to, data);+ }++ pub fn new_table(&mut self, table: Table) {+ self.data.insert(table, BTreeMap::new());+ }++ pub fn tables(&self) -> Vec<Table> {+ self.data.keys().cloned().collect()+ }++ pub fn batch_get(&self, table: &Table, keys: &[Key]) -> Vec<(Key, Value)> {+ match self.data.get(table) {+ None => Vec::new(),+ Some(table) => keys+ .iter()+ .filter_map(|key| {+ table+ .get(key)+ .map(|value| (key.clone(), value.as_ref().clone()))+ })+ .collect(),+ }+ }++ pub fn range_get(+ &self,+ table: &Table,+ range: Range<Bound<Key>>,+ limit: Option<usize>,+ ) -> Vec<(Key, Value)> {+ match self.data.get(table) {+ None => Vec::new(),+ Some(table) => match limit {+ None => table+ .range((range.start, range.end))+ .map(|(key, value)| (key.as_ref().clone(), value.as_ref().clone()))+ .collect(),+ Some(limit) => table+ .range((range.start, range.end))+ .take(limit)+ .map(|(key, value)| (key.as_ref().clone(), value.as_ref().clone()))+ .collect(),+ },+ }+ }+}++#[derive(Debug)]+pub struct StoredSnapshot {+ pub meta: SnapshotMeta<NodeId, BasicNode>,++ /// The data of the state machine at the time of this snapshot.+ pub data: Vec<u8>,+}++#[derive(+ serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Default, Clone,+)]+pub struct StateMachineData {+ #[bincode(with_serde)]+ pub last_applied_log: Option<LogId<NodeId>>,+ #[bincode(with_serde)]+ pub last_membership: StoredMembership<NodeId, BasicNode>,++ /// Application data.+ pub db: Db,+}++#[derive(Debug, Default)]+pub struct StateMachineStore {+ pub state_machine: RwLock<StateMachineData>,+ snapshot_idx: Arc<Mutex<u64>>,++ /// The last received snapshot.+ current_snapshot: RwLock<Option<StoredSnapshot>>,+}++impl RaftSnapshotBuilder<TypeConfig> for Arc<StateMachineStore> {+ #[tracing::instrument(level = "trace", skip(self))]+ async fn build_snapshot(&mut self) -> Result<Snapshot<TypeConfig>, StorageError<NodeId>> {+ let data;+ let last_applied_log;+ let last_membership;++ {+ // Serialize the data of the state machine.+ let state_machine = self.state_machine.read().await;+ let encoded = bincode::encode_to_vec(&*state_machine, bincode::config::standard())+ .map_err(|e| StorageIOError::read_state_machine(&e))?;+ data = encoded;++ last_applied_log = state_machine.last_applied_log;+ last_membership = state_machine.last_membership.clone();+ }++ let snapshot_idx = {+ let mut l = self.snapshot_idx.lock().unwrap();+ *l += 1;+ *l+ };++ let snapshot_id = if let Some(last) = last_applied_log {+ format!("{}-{}-{}", last.leader_id, last.index, snapshot_idx)+ } else {+ format!("--{}", snapshot_idx)+ };++ let meta = SnapshotMeta {+ last_log_id: last_applied_log,+ last_membership,+ snapshot_id,+ };++ let snapshot = StoredSnapshot {+ meta: meta.clone(),+ data: data.clone(),+ };++ {+ let mut current_snapshot = self.current_snapshot.write().await;+ *current_snapshot = Some(snapshot);+ }++ Ok(Snapshot {+ meta,+ snapshot: Box::new(Cursor::new(data)),+ })+ }+}++impl RaftStateMachine<TypeConfig> for Arc<StateMachineStore> {+ type SnapshotBuilder = Self;++ async fn applied_state(+ &mut self,+ ) -> Result<(Option<LogId<NodeId>>, StoredMembership<NodeId, BasicNode>), StorageError<NodeId>>+ {+ let state_machine = self.state_machine.read().await;+ Ok((+ state_machine.last_applied_log,+ state_machine.last_membership.clone(),+ ))+ }++ #[tracing::instrument(level = "trace", skip(self, entries))]+ async fn apply<I>(&mut self, entries: I) -> Result<Vec<Response>, StorageError<NodeId>>+ where+ I: IntoIterator<Item = Entry<TypeConfig>> + Send,+ {+ let mut res = Vec::new();+ let mut sm = self.state_machine.write().await;++ for entry in entries {+ tracing::debug!(%entry.log_id, "replicate to sm");++ if let Some(ref last) = sm.last_applied_log {+ if last >= &entry.log_id {+ res.push(Response::Empty);+ continue;+ }+ }++ sm.last_applied_log = Some(entry.log_id);++ match entry.payload {+ EntryPayload::Blank => res.push(Response::Empty),+ EntryPayload::Normal(ref req) => match req {+ Request::Set(api::Set { table, key, value }) => {+ sm.db.set(table.clone(), key.clone(), value.clone());+ res.push(Response::Set(Ok(())))+ }+ Request::BatchSet(api::BatchSet { table, values }) => {+ sm.db.batch_set(table.clone(), values.as_ref().clone());+ res.push(Response::Set(Ok(())))+ }+ Request::Upsert(api::Upsert {+ table,+ key,+ value,+ upsert_fn,+ }) => res.push(Response::Upsert(Ok(sm.db.upsert(+ table.clone(),+ upsert_fn,+ key.clone(),+ value.clone(),+ )))),+ Request::BatchUpsert(api::BatchUpsert {+ table,+ upsert_fn,+ values,+ }) => res.push(Response::BatchUpsert(Ok(sm.db.batch_upsert(+ table.clone(),+ upsert_fn,+ values.as_ref().clone(),+ )))),+ Request::CreateTable(api::CreateTable { table }) => {+ sm.db.new_table(table.clone());+ res.push(Response::CreateTable(Ok(())))+ }+ Request::DropTable(api::DropTable { table }) => {+ sm.db.drop_table(table);+ res.push(Response::DropTable(Ok(())))+ }+ Request::AllTables(api::AllTables) => {+ res.push(Response::AllTables(Ok(sm.db.tables())))+ }+ Request::CloneTable(api::CloneTable { from, to }) => {+ sm.db.clone_table(from, to.clone());+ res.push(Response::CloneTable(Ok(())))+ }+ },+ EntryPayload::Membership(ref mem) => {+ sm.last_membership = StoredMembership::new(Some(entry.log_id), mem.clone());+ res.push(Response::Empty)+ }+ };+ }+ Ok(res)+ }++ #[tracing::instrument(level = "trace", skip(self))]+ async fn begin_receiving_snapshot(+ &mut self,+ ) -> Result<Box<<TypeConfig as RaftTypeConfig>::SnapshotData>, StorageError<NodeId>> {+ Ok(Box::new(Cursor::new(Vec::new())))+ }++ #[tracing::instrument(level = "trace", skip(self, snapshot))]+ async fn install_snapshot(+ &mut self,+ meta: &SnapshotMeta<NodeId, BasicNode>,+ snapshot: Box<<TypeConfig as RaftTypeConfig>::SnapshotData>,+ ) -> Result<(), StorageError<NodeId>> {+ tracing::info!(+ { snapshot_size = snapshot.get_ref().len() },+ "decoding snapshot for installation"+ );++ let new_snapshot = StoredSnapshot {+ meta: meta.clone(),+ data: snapshot.into_inner(),+ };++ // Update the state machine.+ {+ let (data, _): (Db, _) =+ bincode::decode_from_slice(&new_snapshot.data, bincode::config::standard())+ .map_err(|e| {+ StorageIOError::read_snapshot(Some(new_snapshot.meta.signature()), &e)+ })?;++ let mut state_machine = self.state_machine.write().await;+ state_machine.db = data;+ state_machine.last_applied_log = meta.last_log_id;+ state_machine.last_membership = meta.last_membership.clone();+ }++ // Update current snapshot.+ let mut current_snapshot = self.current_snapshot.write().await;+ *current_snapshot = Some(new_snapshot);+ Ok(())+ }++ #[tracing::instrument(level = "trace", skip(self))]+ async fn get_current_snapshot(+ &mut self,+ ) -> Result<Option<Snapshot<TypeConfig>>, StorageError<NodeId>> {+ match &*self.current_snapshot.read().await {+ Some(snapshot) => {+ let data = snapshot.data.clone();+ Ok(Some(Snapshot {+ meta: snapshot.meta.clone(),+ snapshot: Box::new(Cursor::new(data)),+ }))+ }+ None => Ok(None),+ }+ }++ async fn get_snapshot_builder(&mut self) -> Self::SnapshotBuilder {+ Arc::clone(self)+ }+}++#[cfg(test)]+mod tests {+ use openraft::testing::{StoreBuilder, Suite};++ type LogStore = crate::ampc::dht::log_store::LogStore<TypeConfig>;++ use super::*;++ struct MemStoreBuilder {}++ impl StoreBuilder<TypeConfig, LogStore, Arc<StateMachineStore>, ()> for MemStoreBuilder {+ async fn build(+ &self,+ ) -> Result<((), LogStore, Arc<StateMachineStore>), StorageError<NodeId>> {+ let log_store = LogStore::default();+ let sm = Arc::new(StateMachineStore::default());++ Ok(((), log_store, sm))+ }+ }++ #[test]+ pub fn test_raft_impl() -> Result<(), StorageError<NodeId>> {+ Suite::test_all(MemStoreBuilder {})?;+ Ok(())+ }+}diff --git a/crates/core/src/ampc/dht/upsert.rs b/crates/core/src/ampc/dht/upsert.rsnew file mode 100644index 00000000..712f339a--- /dev/null+++ b/crates/core/src/ampc/dht/upsert.rs@@ -0,0 +1,103 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use enum_dispatch::enum_dispatch;++use crate::hyperloglog::HyperLogLog;++use super::Value;++#[derive(+ Debug, Clone, Copy, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,+)]+pub enum UpsertAction {+ Merged,+ NoChange,+ Inserted,+}++#[enum_dispatch]+pub trait UpsertFn {+ fn upsert(&self, old: Value, new: Value) -> Value;+}++#[enum_dispatch(UpsertFn)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub enum UpsertEnum {+ HyperLogLog64Upsert,+ U64Add,+ F64Add,+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct HyperLogLog64Upsert;++impl UpsertFn for HyperLogLog64Upsert {+ fn upsert(&self, old: Value, new: Value) -> Value {+ let (mut old, _) = bincode::decode_from_slice::<HyperLogLog<64>, _>(+ old.as_bytes(),+ bincode::config::standard(),+ )+ .unwrap();+ let (new, _) = bincode::decode_from_slice::<HyperLogLog<64>, _>(+ new.as_bytes(),+ bincode::config::standard(),+ )+ .unwrap();++ old.merge(&new);++ bincode::encode_to_vec(&old, bincode::config::standard())+ .unwrap()+ .into()+ }+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct U64Add;++impl UpsertFn for U64Add {+ fn upsert(&self, old: Value, new: Value) -> Value {+ let (old, _) =+ bincode::decode_from_slice::<u64, _>(old.as_bytes(), bincode::config::standard())+ .unwrap();+ let (new, _) =+ bincode::decode_from_slice::<u64, _>(new.as_bytes(), bincode::config::standard())+ .unwrap();++ bincode::encode_to_vec(old + new, bincode::config::standard())+ .unwrap()+ .into()+ }+}++#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub struct F64Add;++impl UpsertFn for F64Add {+ fn upsert(&self, old: Value, new: Value) -> Value {+ let (old, _) =+ bincode::decode_from_slice::<f64, _>(old.as_bytes(), bincode::config::standard())+ .unwrap();+ let (new, _) =+ bincode::decode_from_slice::<f64, _>(new.as_bytes(), bincode::config::standard())+ .unwrap();++ bincode::encode_to_vec(old + new, bincode::config::standard())+ .unwrap()+ .into()+ }+}diff --git a/crates/core/src/ampc/dht_conn.rs b/crates/core/src/ampc/dht_conn.rsnew file mode 100644index 00000000..c34c489d--- /dev/null+++ b/crates/core/src/ampc/dht_conn.rs@@ -0,0 +1,528 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{collections::BTreeMap, net::SocketAddr, pin::Pin};++use super::{+ block_on,+ dht::{self, upsert::UpsertEnum, UpsertAction},+};++use crate::Result;++pub trait DhtTables+where+ Self: Clone + bincode::Encode + bincode::Decode + Send + Sync,+{+ fn drop_tables(&self);+ fn next(&self) -> Self;+ fn cleanup_prev_tables(&self);+}++// TODO: this could be a derive proc macro instead+macro_rules! impl_dht_tables {+ ($struct:ty, [$($field:ident),*$(,)?]) => {+ #[allow(unused_imports)]+ use $crate::ampc::dht_conn::DhtTable as _;++ impl $crate::ampc::dht_conn::DhtTables for $struct {+ fn drop_tables(&self) {+ $(self.$field.drop_table();)*+ }++ fn next(&self) -> Self {+ Self {+ $($field: self.$field.next(),)*+ }+ }++ fn cleanup_prev_tables(&self) {+ $(+ let tables = $crate::ampc::block_on(self.$field.client().all_tables()).unwrap();++ for table in tables {+ if table.as_str().starts_with(&self.$field.table().prefix()) {+ $crate::ampc::block_on(self.$field.client().drop_table(table)).unwrap();+ }+ }+ )*+ }+ }+ };+}++use futures::{Stream, StreamExt};+pub(crate) use impl_dht_tables;++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, Debug)]+pub struct Table {+ prefix: String,+ round: u64,+}++impl Table {+ pub fn new<S: AsRef<str>>(prefix: S) -> Self {+ Self {+ prefix: prefix.as_ref().to_string(),+ round: 0,+ }+ }++ pub fn prefix(&self) -> &str {+ &self.prefix+ }++ fn dht(&self) -> dht::Table {+ format!("{}-{}", self.prefix, self.round).into()+ }++ fn next(&self) -> Self {+ Self {+ prefix: self.prefix.clone(),+ round: self.round + 1,+ }+ }+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug)]+pub struct DefaultDhtTable<K, V> {+ table: Table,+ client: dht::Client,+ _maker: std::marker::PhantomData<(K, V)>,+}++impl<K, V> Clone for DefaultDhtTable<K, V> {+ fn clone(&self) -> Self {+ Self {+ table: self.table.clone(),+ client: self.client.clone(),+ _maker: std::marker::PhantomData,+ }+ }+}+impl<K, V> DefaultDhtTable<K, V>+where+ K: bincode::Encode + bincode::Decode,+ V: bincode::Encode + bincode::Decode,+{+ pub fn new<S: AsRef<str>>(members: &[(dht::ShardId, SocketAddr)], prefix: S) -> Self {+ Self {+ table: Table::new(prefix),+ client: dht::Client::new(members),+ _maker: std::marker::PhantomData,+ }+ }++ pub fn shards(&self) -> &BTreeMap<dht::ShardId, dht::Shard> {+ self.client.shards()+ }+}++pub trait DhtTable: Clone + bincode::Encode + bincode::Decode {+ type Key: bincode::Encode + bincode::Decode;+ type Value: bincode::Encode + bincode::Decode;++ fn client(&self) -> &dht::Client;+ fn table(&self) -> &Table;+ fn next(&self) -> Self;++ fn get(&self, key: Self::Key) -> Option<Self::Value> {+ let key = bincode::encode_to_vec(&key, bincode::config::standard()).unwrap();++ block_on(self.client().get(self.table().dht(), key.into()))+ .unwrap()+ .map(|v| {+ let (k, _) =+ bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).unwrap();+ k+ })+ }++ fn batch_get(&self, keys: Vec<Self::Key>) -> Vec<(Self::Key, Self::Value)> {+ let keys: Vec<dht::Key> = keys+ .into_iter()+ .map(|k| {+ bincode::encode_to_vec(&k, bincode::config::standard())+ .unwrap()+ .into()+ })+ .collect::<Vec<_>>();+ let values = block_on(self.client().batch_get(self.table().dht(), keys)).unwrap();++ values+ .into_iter()+ .map(|(k, v)| {+ let (k, _) =+ bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).unwrap();+ let (v, _) =+ bincode::decode_from_slice(v.as_bytes(), bincode::config::standard()).unwrap();++ (k, v)+ })+ .collect()+ }++ fn set(&self, key: Self::Key, value: Self::Value) {+ let key = bincode::encode_to_vec(&key, bincode::config::standard()).unwrap();+ let value = bincode::encode_to_vec(&value, bincode::config::standard()).unwrap();++ block_on(+ self.client()+ .set(self.table().dht(), key.into(), value.into()),+ )+ .unwrap();+ }++ fn batch_set(&self, pairs: Vec<(Self::Key, Self::Value)>) {+ let pairs: Vec<(dht::Key, dht::Value)> = pairs+ .into_iter()+ .map(|(k, v)| {+ (+ bincode::encode_to_vec(&k, bincode::config::standard())+ .unwrap()+ .into(),+ bincode::encode_to_vec(&v, bincode::config::standard())+ .unwrap()+ .into(),+ )+ })+ .collect();++ block_on(self.client().batch_set(self.table().dht(), pairs)).unwrap();+ }++ fn upsert<F: Into<UpsertEnum>>(+ &self,+ upsert: F,+ key: Self::Key,+ value: Self::Value,+ ) -> UpsertAction {+ let key = bincode::encode_to_vec(&key, bincode::config::standard()).unwrap();+ let value = bincode::encode_to_vec(&value, bincode::config::standard()).unwrap();++ block_on(+ self.client()+ .upsert(self.table().dht(), upsert, key.into(), value.into()),+ )+ .unwrap()+ }++ fn batch_upsert<F: Into<UpsertEnum> + Clone>(+ &self,+ upsert: F,+ pairs: Vec<(Self::Key, Self::Value)>,+ ) -> Vec<(Self::Key, UpsertAction)> {+ let pairs: Vec<(dht::Key, dht::Value)> = pairs+ .into_iter()+ .map(|(k, v)| {+ (+ bincode::encode_to_vec(&k, bincode::config::standard())+ .unwrap()+ .into(),+ bincode::encode_to_vec(&v, bincode::config::standard())+ .unwrap()+ .into(),+ )+ })+ .collect();++ block_on(+ self.client()+ .batch_upsert(self.table().dht(), upsert, pairs),+ )+ .unwrap()+ .into_iter()+ .map(|(k, did_upsert)| {+ let (k, _) =+ bincode::decode_from_slice(k.as_bytes(), bincode::config::standard()).unwrap();+ (k, did_upsert)+ })+ .collect()+ }++ fn init_from(&self, prev: &DefaultDhtTable<Self::Key, Self::Value>) {+ block_on(+ self.client()+ .clone_table(prev.table().dht(), self.table().dht()),+ )+ .unwrap();+ }++ fn drop_table(&self) {+ block_on(self.client().drop_table(self.table().dht())).unwrap();+ }++ fn raw_iter(&self) -> impl Iterator<Item = (dht::Key, dht::Value)> + '_ {+ let s = self.client().stream(self.table().dht());+ DhtTableIterator::new(s)+ }++ fn iter(&self) -> impl Iterator<Item = (Self::Key, Self::Value)> + '_ {+ self.raw_iter().map(|(key, value)| {+ let (key, _) =+ bincode::decode_from_slice(key.as_bytes(), bincode::config::standard()).unwrap();+ let (value, _) =+ bincode::decode_from_slice(value.as_bytes(), bincode::config::standard()).unwrap();++ (key, value)+ })+ }+}++struct DhtTableIterator<S> {+ stream: Pin<Box<S>>,+ batch: Vec<(dht::Key, dht::Value)>,+}++impl<S> DhtTableIterator<S> {+ fn new(stream: S) -> Self {+ Self {+ stream: Box::pin(stream),+ batch: Vec::new(),+ }+ }+}++impl<'a, S> Iterator for DhtTableIterator<S>+where+ S: Stream<Item = Result<(dht::Key, dht::Value)>> + 'a,+{+ type Item = (dht::Key, dht::Value);++ fn next(&mut self) -> Option<Self::Item> {+ if self.batch.is_empty() {+ self.batch = block_on(iter_batch(&mut self.stream));+ }++ self.batch.pop()+ }+}++async fn iter_batch<S>(stream: &mut Pin<Box<S>>) -> Vec<(dht::Key, dht::Value)>+where+ S: Stream<Item = Result<(dht::Key, dht::Value)>>,+{+ let mut res = Vec::new();+ let mut count = 0;++ while let Some(item) = stream.next().await {+ match item {+ Ok((k, v)) => {+ res.push((k, v));+ count += 1;+ }+ Err(_) => break,+ }++ if count >= 1024 {+ break;+ }+ }++ res+}++impl<K, V> DhtTable for DefaultDhtTable<K, V>+where+ K: bincode::Encode + bincode::Decode,+ V: bincode::Encode + bincode::Decode,+{+ type Key = K;+ type Value = V;++ fn client(&self) -> &dht::Client {+ &self.client+ }++ fn table(&self) -> &Table {+ &self.table+ }++ fn next(&self) -> DefaultDhtTable<Self::Key, Self::Value> {+ let new = Self {+ table: self.table().next(),+ client: self.client().clone(),+ _maker: std::marker::PhantomData,+ };++ new.init_from(self);++ new+ }+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]+pub struct DhtConn<T> {+ prev: T,+ next: T,+}++impl<T> DhtConn<T>+where+ T: DhtTables,+{+ pub fn new(initial: T) -> Self {+ let next = initial.next();++ Self {+ prev: initial,+ next,+ }+ }++ pub(super) fn cleanup_prev_tables(&self) {+ self.prev.cleanup_prev_tables();+ }++ pub(super) fn next_round(&mut self) {+ self.prev.drop_tables();+ self.prev = self.next.clone();++ self.next = self.prev.next();+ }++ pub fn prev(&self) -> &T {+ &self.prev+ }++ pub fn next(&self) -> &T {+ &self.next+ }++ pub fn take_prev(self) -> T {+ self.prev+ }+}++#[cfg(test)]+mod tests {+ use openraft::error::InitializeError;+ use tracing_test::traced_test;++ use self::dht::{upsert, BasicNode};++ use super::*;++ #[derive(+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ PartialOrd,+ Ord,+ )]+ struct Id(u64);+ #[derive(+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ )]+ struct Counter(u64);++ #[derive(Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+ struct Tables {+ id: DefaultDhtTable<Id, Counter>,+ }++ impl_dht_tables!(Tables, [id]);++ pub fn start_dht_background() -> SocketAddr {+ let (tx, rx) = crossbeam_channel::unbounded();+ std::thread::spawn(move || {+ let rt = tokio::runtime::Runtime::new().unwrap();+ rt.block_on(async {+ let (raft, server, addr) = dht::tests::server(1).await.unwrap();++ let members: BTreeMap<u64, _> = vec![(1, addr)]+ .into_iter()+ .map(|(id, addr)| (id, BasicNode::new(addr)))+ .collect();++ if let Err(e) = raft.initialize(members.clone()).await {+ match e {+ openraft::error::RaftError::APIError(e) => match e {+ InitializeError::NotAllowed(_) => {}+ InitializeError::NotInMembers(_) => panic!("{:?}", e),+ },+ openraft::error::RaftError::Fatal(_) => panic!("{:?}", e),+ }+ };++ tx.send(addr).unwrap();++ loop {+ server.accept().await.unwrap();+ }+ })+ });++ rx.recv().unwrap()+ }++ #[test]+ #[traced_test]+ fn test_dht_conn() -> anyhow::Result<()> {+ let addr = start_dht_background();++ let tables = Tables {+ id: DefaultDhtTable::new(&[(1.into(), addr)], "id"),+ };++ tables.id.set(Id(0), Counter(0));++ assert_eq!(tables.id.get(Id(0)), Some(Counter(0)));++ tables+ .id+ .batch_set(vec![(Id(1), Counter(0)), (Id(2), Counter(0))]);++ let mut res = tables.id.batch_get(vec![Id(1), Id(2)]);+ res.sort_by(|(a, _), (b, _)| a.cmp(&b));++ assert_eq!(res, vec![(Id(1), Counter(0)), (Id(2), Counter(0))]);++ tables.id.upsert(upsert::U64Add, Id(0), Counter(1));+ assert_eq!(tables.id.get(Id(0)), Some(Counter(1)));++ tables.id.batch_upsert(+ upsert::U64Add,+ vec![(Id(1), Counter(1)), (Id(2), Counter(1))],+ );++ let mut res = tables.id.batch_get(vec![Id(0), Id(1), Id(2)]);+ res.sort_by(|(a, _), (b, _)| a.cmp(&b));++ assert_eq!(+ res,+ vec![+ (Id(0), Counter(1)),+ (Id(1), Counter(1)),+ (Id(2), Counter(1))+ ]+ );++ Ok(())+ }+}diff --git a/crates/core/src/ampc/finisher.rs b/crates/core/src/ampc/finisher.rsnew file mode 100644index 00000000..72a9dd92--- /dev/null+++ b/crates/core/src/ampc/finisher.rs@@ -0,0 +1,23 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licensepub trait Finisher {++use super::prelude::Job;++pub trait Finisher {+ type Job: Job;++ fn is_finished(&self, dht: &<<Self as Finisher>::Job as Job>::DhtTables) -> bool;+}diff --git a/crates/core/src/ampc/job.rs b/crates/core/src/ampc/job.rsnew file mode 100644index 00000000..d7c2a6c4--- /dev/null+++ b/crates/core/src/ampc/job.rs@@ -0,0 +1,31 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses++use super::{DhtTables, Mapper, Worker};++pub trait Job+where+ Self: bincode::Encode + bincode::Decode + std::fmt::Debug + Clone + Send + Sync,+{+ type DhtTables: DhtTables;+ type Worker: Worker<Job = Self>;+ type Mapper: Mapper<Job = Self>;++ #[allow(unused_variables)] // reason="`worker` might be used by implementors"+ fn is_schedulable(&self, worker: &<<Self as Job>::Worker as Worker>::Remote) -> bool {+ true+ }+}diff --git a/crates/core/src/ampc/mapper.rs b/crates/core/src/ampc/mapper.rsnew file mode 100644index 00000000..6150cb1b--- /dev/null+++ b/crates/core/src/ampc/mapper.rs@@ -0,0 +1,28 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/license++use super::{prelude::Job, DhtConn};++pub trait Mapper: bincode::Encode + bincode::Decode + Send + Sync + Clone {+ type Job: Job<Mapper = Self>;++ fn map(+ &self,+ job: Self::Job,+ worker: &<<Self as Mapper>::Job as Job>::Worker,+ dht: &DhtConn<<<Self as Mapper>::Job as Job>::DhtTables>,+ );+}diff --git a/crates/core/src/ampc/mod.rs b/crates/core/src/ampc/mod.rsnew file mode 100644index 00000000..a2e58c23--- /dev/null+++ b/crates/core/src/ampc/mod.rs@@ -0,0 +1,120 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use self::{job::Job, worker::WorkerRef};+use crate::distributed::sonic;+use std::time::Duration;++mod coordinator;+pub mod dht;+pub mod dht_conn;+mod finisher;+mod job;+mod mapper;+pub mod prelude;+mod server;+mod setup;+mod worker;++use self::prelude::*;++pub use coordinator::Coordinator;+pub use dht_conn::{DefaultDhtTable, DhtConn, DhtTable, DhtTables, Table};+pub use server::Server;+pub use worker::{Message, RequestWrapper, Worker};++static TOKIO_RUNTIME: once_cell::sync::Lazy<tokio::runtime::Runtime> =+ once_cell::sync::Lazy::new(|| {+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ });++pub fn block_on<F: std::future::Future>(f: F) -> F::Output {+ TOKIO_RUNTIME.block_on(f)+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]+pub enum CoordReq<J, M, T> {+ CurrentJob,+ ScheduleJob { job: J, mapper: M },+ Setup { dht: DhtConn<T> },+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub enum CoordResp<J> {+ CurrentJob(Option<J>),+ ScheduleJob(()),+ Setup(()),+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]+pub enum Req<J, M, R, T> {+ Coordinator(CoordReq<J, M, T>),+ User(R),+}++type JobReq<J> =+ Req<J, <J as Job>::Mapper, <<J as Job>::Worker as Worker>::Request, <J as Job>::DhtTables>;++type JobResp<J> = Resp<J, <<J as Job>::Worker as Worker>::Response>;++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]+pub enum Resp<J, R> {+ Coordinator(CoordResp<J>),+ User(R),+}++type JobDht<J> = DhtConn<<J as Job>::DhtTables>;++type JobConn<J> = sonic::Connection<JobReq<J>, JobResp<J>>;++#[must_use = "this `JobScheduled` may not have scheduled the job on any worker"]+enum JobScheduled {+ Success(WorkerRef),+ NoAvailableWorkers,+}++struct ExponentialBackoff {+ min: Duration,+ max: Duration,+ factor: f64,+ attempts: usize,+}++impl ExponentialBackoff {+ fn new(min: Duration, max: Duration, factor: f64) -> Self {+ Self {+ min,+ max,+ factor,+ attempts: 0,+ }+ }+}++impl ExponentialBackoff {+ fn next(&mut self) -> Duration {+ let duration = self.min.mul_f64(self.factor.powi(self.attempts as i32));+ self.attempts += 1;+ duration.min(self.max)+ }++ fn success(&mut self) {+ self.attempts = 0;+ }+}diff --git a/crates/core/src/ampc/prelude.rs b/crates/core/src/ampc/prelude.rsnew file mode 100644index 00000000..3762d63b--- /dev/null+++ b/crates/core/src/ampc/prelude.rs@@ -0,0 +1,26 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/++pub(crate) use super::dht_conn::impl_dht_tables;+pub(crate) use super::worker::impl_worker;++pub use super::dht_conn::{DhtTable, DhtTables};+pub use super::finisher::Finisher;+pub use super::job::Job;+pub use super::mapper::Mapper;+pub use super::setup::Setup;+pub use super::worker::RemoteWorker;+pub use super::worker::{Message, Worker};diff --git a/crates/core/src/ampc/server.rs b/crates/core/src/ampc/server.rsnew file mode 100644index 00000000..1cc1ab74--- /dev/null+++ b/crates/core/src/ampc/server.rs@@ -0,0 +1,100 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use std::sync::{Arc, Mutex};++use tokio::net::ToSocketAddrs;++use crate::distributed::sonic;+use crate::Result;++use super::{block_on, CoordReq, CoordResp, JobDht, JobReq, JobResp, Mapper, Req, Resp, Worker};++pub struct Server<W>+where+ W: Worker,+{+ dht: Option<Arc<JobDht<W::Job>>>,+ worker: Arc<W>,+ current_job: Arc<Mutex<Option<W::Job>>>,+ conn: sonic::Server<JobReq<W::Job>, JobResp<W::Job>>,+}++impl<W> Server<W>+where+ W: Worker + 'static,+{+ fn handle(&mut self) -> Result<()> {+ let req = block_on(self.conn.accept())?;++ match req.body().clone() {+ Req::Coordinator(coord_req) => {+ let res = match coord_req {+ CoordReq::CurrentJob => Resp::Coordinator(CoordResp::CurrentJob(+ self.current_job.lock().unwrap().clone(),+ )),+ CoordReq::ScheduleJob { job, mapper } => {+ *self.current_job.lock().unwrap() = Some(job.clone());+ let worker = Arc::clone(&self.worker);+ let dht = self.dht.clone();++ let current_job = Arc::clone(&self.current_job);+ std::thread::spawn(move || {+ mapper.map(job.clone(), &worker, dht.as_ref().expect("DHT not set"));+ current_job.lock().unwrap().take();+ });++ Resp::Coordinator(CoordResp::ScheduleJob(()))+ }+ CoordReq::Setup { dht } => {+ self.dht = Some(Arc::new(dht));+ Resp::Coordinator(CoordResp::Setup(()))+ }+ };++ block_on(req.respond(res))?;+ }+ Req::User(user_req) => {+ let worker = Arc::clone(&self.worker);++ std::thread::spawn(move || {+ let res = Resp::User(worker.handle(user_req.clone()));+ block_on(req.respond(res)).unwrap();+ });+ }+ };++ Ok(())+ }++ pub fn bind(addr: impl ToSocketAddrs, worker: W) -> Result<Server<W>> {+ let worker = Arc::new(worker);+ let conn = block_on(sonic::Server::bind(addr))?;++ Ok(Server {+ dht: None,+ worker,+ current_job: Arc::new(Mutex::new(None)),+ conn,+ })+ }++ pub fn run(&mut self) -> Result<()> {+ loop {+ self.handle()?;+ }+ }+}diff --git a/crates/core/src/ampc/setup.rs b/crates/core/src/ampc/setup.rsnew file mode 100644index 00000000..14ef5761--- /dev/null+++ b/crates/core/src/ampc/setup.rs@@ -0,0 +1,28 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/license++use super::DhtConn;++pub trait Setup {+ type DhtTables;++ fn init_dht(&self) -> DhtConn<Self::DhtTables>;+ #[allow(unused_variables)] // reason = "dht might be used by implementors"+ fn setup_round(&self, dht: &Self::DhtTables) {}+ fn setup_first_round(&self, dht: &Self::DhtTables) {+ self.setup_round(dht);+ }+}diff --git a/crates/core/src/ampc/worker.rs b/crates/core/src/ampc/worker.rsnew file mode 100644index 00000000..7bf825e4--- /dev/null+++ b/crates/core/src/ampc/worker.rs@@ -0,0 +1,227 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use std::net::SocketAddr;++use super::{+ block_on, CoordReq, CoordResp, DhtConn, Job, JobConn, JobReq, JobResp, Req, Resp, Server,+};+use crate::Result;+use anyhow::anyhow;+use tokio::net::ToSocketAddrs;++pub trait Worker: Send + Sync {+ type Remote: RemoteWorker<Job = Self::Job>;++ type Request: bincode::Encode + bincode::Decode + Clone + Send + Sync;+ type Response: bincode::Encode + bincode::Decode + Send + Sync;+ type Job: Job<Worker = Self>;++ fn handle(&self, req: Self::Request) -> Self::Response;++ fn bind(self, addr: impl ToSocketAddrs) -> Result<Server<Self>>+ where+ Self: Sized + 'static,+ {+ Server::bind(addr, self)+ }++ fn run(self, addr: impl ToSocketAddrs) -> Result<()>+ where+ Self: Sized + 'static,+ {+ self.bind(addr)?.run()+ }+}++pub trait Message<W: Worker>: std::fmt::Debug + Clone {+ type Response;+ fn handle(self, worker: &W) -> Self::Response;+}++pub trait RemoteWorker+where+ Self: Send + Sync,+{+ type Job: Job;++ fn remote_addr(&self) -> SocketAddr;++ fn schedule_job(&self, job: &Self::Job, mapper: <Self::Job as Job>::Mapper) -> Result<()> {+ self.send_raw(&JobReq::Coordinator(CoordReq::ScheduleJob {+ job: job.clone(),+ mapper,+ }))?;++ Ok(())+ }++ fn send_dht(&self, dht: &DhtConn<<Self::Job as Job>::DhtTables>) -> Result<()> {+ self.send_raw(&JobReq::Coordinator(CoordReq::Setup { dht: dht.clone() }))?;++ Ok(())+ }++ fn current_job(&self) -> Result<Option<Self::Job>> {+ let req = JobReq::Coordinator(CoordReq::CurrentJob);+ let res = self.send_raw(&req)?;++ match res {+ Resp::Coordinator(CoordResp::CurrentJob(job)) => Ok(job),+ _ => Err(anyhow!("unexpected response")),+ }+ }++ fn conn(&self) -> Result<JobConn<Self::Job>> {+ let conn = block_on(JobConn::connect(self.remote_addr()))?;+ Ok(conn)+ }++ fn send_raw(&self, req: &JobReq<Self::Job>) -> Result<JobResp<Self::Job>> {+ let conn = self.conn()?;+ let res = block_on(conn.send(req))?;+ Ok(res)+ }++ fn send<R>(&self, req: R) -> R::Response+ where+ R: RequestWrapper<<Self::Job as Job>::Worker>,+ {+ match self.send_raw(&Req::User(R::wrap(req))).unwrap() {+ Resp::Coordinator(_) => panic!("unexpected coordinator response"),+ Resp::User(res) => R::unwrap_response(res).unwrap(),+ }+ }+}++pub trait RequestWrapper<W: Worker>: Message<W> {+ fn wrap(req: Self) -> W::Request;+ fn unwrap_response(res: W::Response) -> Result<Self::Response>;+}++#[derive(Debug, Clone, Copy, PartialOrd, Ord, PartialEq, Eq)]+pub struct WorkerRef(pub usize);++macro_rules! impl_worker {+ ($job:ident , $remote:ident => $worker:ident, [$($req:ident),*$(,)?]) => {+ mod worker_impl__ {+ #![allow(dead_code)]++ use super::{$worker, $remote, $job, $($req),*};++ use $crate::ampc;+ use $crate::ampc::Message;++ #[derive(Debug, Clone, ::serde::Serialize, ::serde::Deserialize, ::bincode::Decode, ::strum::EnumDiscriminants)]+ pub enum Request {+ $($req($req),)*+ }++ fn req_to_index(req: &Request) -> u32 {+ RequestDiscriminants::from(req) as u32+ }++ fn index_to_req(index: u32) -> RequestDiscriminants {+ match index {+ $(x if x == RequestDiscriminants::$req as u32 => RequestDiscriminants::$req,)*+ _ => panic!("invalid request index"),+ }+ }++ impl ::bincode::Encode for Request {+ fn encode<E: ::bincode::enc::Encoder>(+ &self,+ encoder: &mut E,+ ) -> Result<(), bincode::error::EncodeError> {+ let index = req_to_index(self);+ index.encode(encoder)?;++ match self {+ $(Request::$req(req) => {+ req.encode(encoder)+ })*+ }+ }+ }+++ fn res_to_index(res: &Response) -> u32 {+ ResponseDiscriminants::from(res) as u32+ }++ fn index_to_res(index: u32) -> ResponseDiscriminants {+ match index {+ $(x if x == ResponseDiscriminants::$req as u32 => ResponseDiscriminants::$req,)*+ _ => panic!("invalid response index"),+ }+ }++ #[derive(::serde::Serialize, ::serde::Deserialize, ::bincode::Decode, ::strum::EnumDiscriminants)]+ pub enum Response {+ $($req(<$req as ampc::Message<$worker>>::Response),)*+ }++ impl ::bincode::Encode for Response {+ fn encode<E: ::bincode::enc::Encoder>(+ &self,+ encoder: &mut E,+ ) -> Result<(), bincode::error::EncodeError> {+ let index = res_to_index(self);+ index.encode(encoder)?;++ match self {+ $(Response::$req(res) => {+ res.encode(encoder)+ })*+ }+ }+ }++ impl ampc::Worker for $worker {+ type Remote = $remote;+ type Request = Request;+ type Response = Response;+ type Job = $job;++ fn handle(&self, req: Self::Request) -> Self::Response {+ match req {+ $(Request::$req(req) => Response::$req(req.handle(self)),)*+ }+ }+ }++ $(+ impl ampc::RequestWrapper<$worker> for $req {+ fn wrap(req: Self) -> <$worker as ampc::Worker>::Request {+ <$worker as ampc::Worker>::Request::$req(req)+ }++ fn unwrap_response(res: <$worker as ampc::Worker>::Response) -> anyhow::Result<Self::Response> {+ match res {+ Response::$req(res) => Ok(res),+ _ => Err(anyhow::anyhow!("unexpected response")),+ }+ }+ }+ )*++++ }+ };+}++pub(crate) use impl_worker;diff --git a/crates/core/src/api/autosuggest.rs b/crates/core/src/api/autosuggest.rsindex d6cfe6ce..fbb3e035 100644--- a/crates/core/src/api/autosuggest.rs+++ b/crates/core/src/api/autosuggest.rs@@ -49,7 +49,9 @@ pub struct Suggestion {raw: String,}-#[derive(Debug, serde::Serialize, serde::Deserialize, IntoParams)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, IntoParams,+)]#[serde(rename_all = "camelCase")]pub struct AutosuggestQuery {q: String,diff --git a/crates/core/src/api/search.rs b/crates/core/src/api/search.rsindex 94bf3c20..836e6446 100644--- a/crates/core/src/api/search.rs+++ b/crates/core/src/api/search.rs@@ -33,7 +33,9 @@ use super::State;use axum::{extract, response::IntoResponse};-#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]#[serde(rename_all = "camelCase")]#[schema(title = "SearchQuery", example = json!({"query": "hello world"}))]pub struct ApiSearchQuery {@@ -81,7 +83,9 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {}}-#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]#[serde(tag = "type", rename_all = "camelCase")]pub enum ApiSearchResult {Websites(WebsitesResult),@@ -145,7 +149,9 @@ pub async fn search(}}-#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]pub struct WidgetQuery {pub query: String,}@@ -166,7 +172,9 @@ pub async fn widget(Json(state.searcher.widget(&req.query).await)}-#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]pub struct SidebarQuery {pub query: String,}@@ -187,7 +195,9 @@ pub async fn sidebar(Json(state.searcher.sidebar(&req.query).await)}-#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]pub struct SpellcheckQuery {pub query: String,}@@ -208,7 +218,9 @@ pub async fn spellcheck(Json(state.searcher.spell_check(&req.query))}-#[derive(Debug, serde::Serialize, serde::Deserialize, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct EntityImageParams {pub image_id: String,diff --git a/crates/core/src/api/user_count.rs b/crates/core/src/api/user_count.rsindex 6635b2e6..846bed4f 100644--- a/crates/core/src/api/user_count.rs+++ b/crates/core/src/api/user_count.rs@@ -78,12 +78,12 @@ impl<F: Frequency> InnerUserCount<F> {})}- fn inc<T: serde::Serialize>(&mut self, user_id: &T) -> Result<()> {+ fn inc<T: bincode::Encode>(&mut self, user_id: &T) -> Result<()> {// It is important that we do not store the user_id as it might be sensitive.// It is first hashed with a salt for good measure, and then we only use// the id for a probabilistic count using hyperloglog.self.maybe_reset();- let bytes = bincode::serialize(user_id)?;+ let bytes = bincode::encode_to_vec(user_id, bincode::config::standard())?;let mut hash = [0u8; digest::SHA512_OUTPUT_LEN];pbkdf2::derive(@@ -130,7 +130,7 @@ impl<F: Frequency> UserCount<F> {})}- pub fn inc<T: serde::Serialize>(&self, user_id: &T) -> Result<()> {+ pub fn inc<T: bincode::Encode>(&self, user_id: &T) -> Result<()> {self.inner.lock().unwrap_or_else(|e| e.into_inner())diff --git a/crates/core/src/api/webgraph.rs b/crates/core/src/api/webgraph.rsindex 855c2dc4..4db9628f 100644--- a/crates/core/src/api/webgraph.rs+++ b/crates/core/src/api/webgraph.rs@@ -329,7 +329,7 @@ async fn outgoing_links(.await?)}-#[derive(serde::Serialize, serde::Deserialize, ToSchema)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema)]#[serde(tag = "type", rename_all = "camelCase")]pub enum KnowsHost {Known { host: String },diff --git a/crates/core/src/bangs.rs b/crates/core/src/bangs.rsindex 9ce07919..3d77b61c 100644--- a/crates/core/src/bangs.rs+++ b/crates/core/src/bangs.rs@@ -22,7 +22,7 @@ use std::{};use itertools::intersperse;-use serde::{Deserialize, Serialize};+use url::Url;use utoipa::ToSchema;@@ -30,7 +30,17 @@ use crate::query::parser::Term;pub const BANG_PREFIXES: [char; 2] = ['!', '!'];-#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, ToSchema)]+#[derive(+ Clone,+ Debug,+ PartialEq,+ Eq,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct Bang {#[serde(rename = "c")]@@ -56,9 +66,19 @@ pub struct Bang {}/// Wrapper around `Url` that implements `ToSchema` for `Url`.-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, ToSchema)]+#[derive(+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Clone,+ ToSchema,+)]#[schema(value_type = String, title = "Url")]-pub struct UrlWrapper(Url);+pub struct UrlWrapper(#[bincode(with_serde)] Url);impl DerefMut for UrlWrapper {fn deref_mut(&mut self) -> &mut Self::Target {@@ -80,7 +100,17 @@ impl From<Url> for UrlWrapper {}}-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, ToSchema)]+#[derive(+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Clone,+ ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct BangHit {pub bang: Bang,diff --git a/crates/core/src/bincode_utils.rs b/crates/core/src/bincode_utils.rsnew file mode 100644index 00000000..d41ac1cf--- /dev/null+++ b/crates/core/src/bincode_utils.rs@@ -0,0 +1,899 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++//! This has been copied from `bincode` so that we can derive Clone, serde::Serialize and serde::Deserialize for `Compat`+#[derive(Clone, serde::Serialize, serde::Deserialize, thiserror::Error)]+pub struct SerdeCompat<T>(pub T);++impl<T> bincode::Decode for SerdeCompat<T>+where+ T: serde::de::DeserializeOwned,+{+ fn decode<D: bincode::de::Decoder>(+ decoder: &mut D,+ ) -> Result<Self, bincode::error::DecodeError> {+ let serde_decoder = de_owned::SerdeDecoder { de: decoder };+ T::deserialize(serde_decoder).map(SerdeCompat)+ }+}+impl<'de, T> bincode::BorrowDecode<'de> for SerdeCompat<T>+where+ T: serde::de::DeserializeOwned,+{+ fn borrow_decode<D: bincode::de::BorrowDecoder<'de>>(+ decoder: &mut D,+ ) -> Result<Self, bincode::error::DecodeError> {+ let serde_decoder = de_owned::SerdeDecoder { de: decoder };+ T::deserialize(serde_decoder).map(SerdeCompat)+ }+}++impl<T> bincode::Encode for SerdeCompat<T>+where+ T: serde::Serialize,+{+ fn encode<E: bincode::enc::Encoder>(+ &self,+ encoder: &mut E,+ ) -> Result<(), bincode::error::EncodeError> {+ let serializer = ser::SerdeEncoder { enc: encoder };+ self.0.serialize(serializer)?;+ Ok(())+ }+}++impl<T> core::fmt::Debug for SerdeCompat<T>+where+ T: core::fmt::Debug,+{+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {+ f.debug_tuple("Compat").field(&self.0).finish()+ }+}++impl<T> core::fmt::Display for SerdeCompat<T>+where+ T: core::fmt::Display,+{+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {+ self.0.fmt(f)+ }+}++mod de_owned {+ use bincode::serde::DecodeError as SerdeDecodeError;+ use bincode::{+ de::{Decode, Decoder},+ error::DecodeError,+ };+ use serde::de::*;++ #[inline]+ pub(crate) fn decode_option_variant<D: Decoder>(+ decoder: &mut D,+ type_name: &'static str,+ ) -> Result<Option<()>, DecodeError> {+ let is_some = u8::decode(decoder)?;+ match is_some {+ 0 => Ok(None),+ 1 => Ok(Some(())),+ x => Err(DecodeError::UnexpectedVariant {+ found: x as u32,+ allowed: &bincode::error::AllowedEnumVariants::Range { max: 1, min: 0 },+ type_name,+ }),+ }+ }++ pub(crate) struct SerdeDecoder<'a, DE: Decoder> {+ pub(crate) de: &'a mut DE,+ }++ impl<'a, 'de, DE: Decoder> Deserializer<'de> for SerdeDecoder<'a, DE> {+ type Error = DecodeError;++ fn deserialize_any<V>(self, _: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ Err(SerdeDecodeError::AnyNotSupported.into())+ }++ fn deserialize_bool<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_bool(Decode::decode(&mut self.de)?)+ }++ fn deserialize_i8<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_i8(Decode::decode(&mut self.de)?)+ }++ fn deserialize_i16<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_i16(Decode::decode(&mut self.de)?)+ }++ fn deserialize_i32<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_i32(Decode::decode(&mut self.de)?)+ }++ fn deserialize_i64<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_i64(Decode::decode(&mut self.de)?)+ }++ serde::serde_if_integer128! {+ fn deserialize_i128<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_i128(Decode::decode(&mut self.de)?)+ }+ }++ fn deserialize_u8<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_u8(Decode::decode(&mut self.de)?)+ }++ fn deserialize_u16<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_u16(Decode::decode(&mut self.de)?)+ }++ fn deserialize_u32<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_u32(Decode::decode(&mut self.de)?)+ }++ fn deserialize_u64<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_u64(Decode::decode(&mut self.de)?)+ }++ serde::serde_if_integer128! {+ fn deserialize_u128<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_u128(Decode::decode(&mut self.de)?)+ }+ }++ fn deserialize_f32<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_f32(Decode::decode(&mut self.de)?)+ }++ fn deserialize_f64<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_f64(Decode::decode(&mut self.de)?)+ }++ fn deserialize_char<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_char(Decode::decode(&mut self.de)?)+ }++ #[cfg(feature = "alloc")]+ fn deserialize_str<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_string(Decode::decode(&mut self.de)?)+ }++ #[cfg(not(feature = "alloc"))]+ fn deserialize_str<V>(self, _: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ Err(SerdeDecodeError::CannotBorrowOwnedData.into())+ }++ fn deserialize_string<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_string(Decode::decode(&mut self.de)?)+ }++ fn deserialize_bytes<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_byte_buf(Decode::decode(&mut self.de)?)+ }++ fn deserialize_byte_buf<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_byte_buf(Decode::decode(&mut self.de)?)+ }++ fn deserialize_option<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ let variant = decode_option_variant(&mut self.de, "Option<T>")?;+ if variant.is_some() {+ visitor.visit_some(self)+ } else {+ visitor.visit_none()+ }+ }++ fn deserialize_unit<V>(self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_unit()+ }++ fn deserialize_unit_struct<V>(+ self,+ _name: &'static str,+ visitor: V,+ ) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_unit()+ }++ fn deserialize_newtype_struct<V>(+ self,+ _name: &'static str,+ visitor: V,+ ) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_newtype_struct(self)+ }++ fn deserialize_seq<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ let len = usize::decode(&mut self.de)?;+ self.deserialize_tuple(len, visitor)+ }++ fn deserialize_tuple<V>(mut self, len: usize, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ struct Access<'a, 'b, DE: Decoder> {+ deserializer: &'a mut SerdeDecoder<'b, DE>,+ len: usize,+ }++ impl<'de, 'a, 'b: 'a, DE: Decoder + 'b> SeqAccess<'de> for Access<'a, 'b, DE> {+ type Error = DecodeError;++ fn next_element_seed<T>(&mut self, seed: T) -> Result<Option<T::Value>, DecodeError>+ where+ T: DeserializeSeed<'de>,+ {+ if self.len > 0 {+ self.len -= 1;+ let value = DeserializeSeed::deserialize(+ seed,+ SerdeDecoder {+ de: self.deserializer.de,+ },+ )?;+ Ok(Some(value))+ } else {+ Ok(None)+ }+ }++ fn size_hint(&self) -> Option<usize> {+ Some(self.len)+ }+ }++ visitor.visit_seq(Access {+ deserializer: &mut self,+ len,+ })+ }++ fn deserialize_tuple_struct<V>(+ self,+ _name: &'static str,+ len: usize,+ visitor: V,+ ) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ self.deserialize_tuple(len, visitor)+ }++ fn deserialize_map<V>(mut self, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ struct Access<'a, 'b, DE: Decoder> {+ deserializer: &'a mut SerdeDecoder<'b, DE>,+ len: usize,+ }++ impl<'de, 'a, 'b: 'a, DE: Decoder + 'b> MapAccess<'de> for Access<'a, 'b, DE> {+ type Error = DecodeError;++ fn next_key_seed<K>(&mut self, seed: K) -> Result<Option<K::Value>, DecodeError>+ where+ K: DeserializeSeed<'de>,+ {+ if self.len > 0 {+ self.len -= 1;+ let key = DeserializeSeed::deserialize(+ seed,+ SerdeDecoder {+ de: self.deserializer.de,+ },+ )?;+ Ok(Some(key))+ } else {+ Ok(None)+ }+ }++ fn next_value_seed<V>(&mut self, seed: V) -> Result<V::Value, DecodeError>+ where+ V: DeserializeSeed<'de>,+ {+ let value = DeserializeSeed::deserialize(+ seed,+ SerdeDecoder {+ de: self.deserializer.de,+ },+ )?;+ Ok(value)+ }++ fn size_hint(&self) -> Option<usize> {+ Some(self.len)+ }+ }++ let len = usize::decode(&mut self.de)?;++ visitor.visit_map(Access {+ deserializer: &mut self,+ len,+ })+ }++ fn deserialize_struct<V>(+ self,+ _name: &'static str,+ fields: &'static [&'static str],+ visitor: V,+ ) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ self.deserialize_tuple(fields.len(), visitor)+ }++ fn deserialize_enum<V>(+ self,+ _name: &'static str,+ _variants: &'static [&'static str],+ visitor: V,+ ) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ visitor.visit_enum(self)+ }++ fn deserialize_identifier<V>(self, _visitor: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ Err(SerdeDecodeError::IdentifierNotSupported.into())+ }++ fn deserialize_ignored_any<V>(self, _: V) -> Result<V::Value, Self::Error>+ where+ V: serde::de::Visitor<'de>,+ {+ Err(SerdeDecodeError::IgnoredAnyNotSupported.into())+ }++ fn is_human_readable(&self) -> bool {+ false+ }+ }++ impl<'de, 'a, DE: Decoder> EnumAccess<'de> for SerdeDecoder<'a, DE> {+ type Error = DecodeError;+ type Variant = Self;++ fn variant_seed<V>(mut self, seed: V) -> Result<(V::Value, Self::Variant), Self::Error>+ where+ V: DeserializeSeed<'de>,+ {+ let idx = u32::decode(&mut self.de)?;+ let val = seed.deserialize(idx.into_deserializer())?;+ Ok((val, self))+ }+ }++ impl<'de, 'a, DE: Decoder> VariantAccess<'de> for SerdeDecoder<'a, DE> {+ type Error = DecodeError;++ fn unit_variant(self) -> Result<(), Self::Error> {+ Ok(())+ }++ fn newtype_variant_seed<T>(self, seed: T) -> Result<T::Value, Self::Error>+ where+ T: DeserializeSeed<'de>,+ {+ DeserializeSeed::deserialize(seed, self)+ }++ fn tuple_variant<V>(self, len: usize, visitor: V) -> Result<V::Value, Self::Error>+ where+ V: Visitor<'de>,+ {+ Deserializer::deserialize_tuple(self, len, visitor)+ }++ fn struct_variant<V>(+ self,+ fields: &'static [&'static str],+ visitor: V,+ ) -> Result<V::Value, Self::Error>+ where+ V: Visitor<'de>,+ {+ Deserializer::deserialize_tuple(self, fields.len(), visitor)+ }+ }+}++mod ser {+ use bincode::serde::EncodeError as SerdeEncodeError;+ use bincode::{+ enc::{Encode, Encoder},+ error::EncodeError,+ };+ use serde::ser::*;++ // /// Encode the given value into a `Vec<u8>` with the given `Config`. See the [config] module for more information.+ // ///+ // /// [config]: ../config/index.html+ // #[cfg(feature = "alloc")]+ // #[cfg_attr(docsrs, doc(cfg(feature = "alloc")))]+ // pub fn encode_to_vec<E, C>(val: E, config: C) -> Result<Vec<u8>, EncodeError>+ // where+ // E: Serialize,+ // C: Config,+ // {+ // let mut encoder = bincode::enc::EncoderImpl::new(bincode::VecWriter::default(), config);+ // let serializer = SerdeEncoder { enc: &mut encoder };+ // val.serialize(serializer)?;+ // Ok(encoder.into_writer().collect())+ // }++ // /// Encode the given value into the given slice. Returns the amount of bytes that have been written.+ // ///+ // /// See the [config] module for more information on configurations.+ // ///+ // /// [config]: ../config/index.html+ // pub fn encode_into_slice<E, C>(val: E, dst: &mut [u8], config: C) -> Result<usize, EncodeError>+ // where+ // E: Serialize,+ // C: Config,+ // {+ // let mut encoder =+ // bincode::enc::EncoderImpl::new(bincode::enc::write::SliceWriter::new(dst), config);+ // let serializer = SerdeEncoder { enc: &mut encoder };+ // val.serialize(serializer)?;+ // Ok(encoder.into_writer().bytes_written())+ // }++ // /// Encode the given value into a custom [Writer].+ // ///+ // /// See the [config] module for more information on configurations.+ // ///+ // /// [config]: ../config/index.html+ // pub fn encode_into_writer<E: Serialize, W: Writer, C: Config>(+ // val: E,+ // writer: W,+ // config: C,+ // ) -> Result<(), EncodeError> {+ // let mut encoder = bincode::enc::EncoderImpl::<_, C>::new(writer, config);+ // let serializer = SerdeEncoder { enc: &mut encoder };+ // val.serialize(serializer)?;+ // Ok(())+ // }++ // /// Encode the given value into any type that implements `std::io::Write`, e.g. `std::fs::File`, with the given `Config`.+ // /// See the [config] module for more information.+ // ///+ // /// [config]: ../config/index.html+ // #[cfg_attr(docsrs, doc(cfg(feature = "std")))]+ // #[cfg(feature = "std")]+ // pub fn encode_into_std_write<E: Serialize, C: Config, W: std::io::Write>(+ // val: E,+ // dst: &mut W,+ // config: C,+ // ) -> Result<usize, EncodeError> {+ // let writer = bincode::IoWriter::new(dst);+ // let mut encoder = bincode::enc::EncoderImpl::<_, C>::new(writer, config);+ // let serializer = SerdeEncoder { enc: &mut encoder };+ // val.serialize(serializer)?;+ // Ok(encoder.into_writer().bytes_written())+ // }++ pub(super) struct SerdeEncoder<'a, ENC: Encoder> {+ pub(super) enc: &'a mut ENC,+ }++ impl<'a, ENC> Serializer for SerdeEncoder<'a, ENC>+ where+ ENC: Encoder,+ {+ type Ok = ();++ type Error = EncodeError;++ type SerializeSeq = Self;+ type SerializeTuple = Self;+ type SerializeTupleStruct = Self;+ type SerializeTupleVariant = Self;+ type SerializeMap = Self;+ type SerializeStruct = Self;+ type SerializeStructVariant = Self;++ fn serialize_bool(self, v: bool) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ fn serialize_i8(self, v: i8) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ fn serialize_i16(self, v: i16) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ fn serialize_i32(self, v: i32) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ fn serialize_i64(self, v: i64) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ serde::serde_if_integer128! {+ fn serialize_i128(self, v: i128) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }+ }++ fn serialize_u8(self, v: u8) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ fn serialize_u16(self, v: u16) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ fn serialize_u32(self, v: u32) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ fn serialize_u64(self, v: u64) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ serde::serde_if_integer128! {+ fn serialize_u128(self, v: u128) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }+ }++ fn serialize_f32(self, v: f32) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ fn serialize_f64(self, v: f64) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ fn serialize_char(self, v: char) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ fn serialize_str(self, v: &str) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ fn serialize_bytes(self, v: &[u8]) -> Result<Self::Ok, Self::Error> {+ v.encode(self.enc)+ }++ fn serialize_none(self) -> Result<Self::Ok, Self::Error> {+ 0u8.encode(self.enc)+ }++ fn serialize_some<T>(mut self, value: &T) -> Result<Self::Ok, Self::Error>+ where+ T: ?Sized + Serialize,+ {+ 1u8.encode(&mut self.enc)?;+ value.serialize(self)+ }++ fn serialize_unit(self) -> Result<Self::Ok, Self::Error> {+ Ok(())+ }++ fn serialize_unit_struct(self, _name: &'static str) -> Result<Self::Ok, Self::Error> {+ Ok(())+ }++ fn serialize_unit_variant(+ self,+ _name: &'static str,+ variant_index: u32,+ _variant: &'static str,+ ) -> Result<Self::Ok, Self::Error> {+ variant_index.encode(self.enc)+ }++ fn serialize_newtype_struct<T>(+ self,+ _name: &'static str,+ value: &T,+ ) -> Result<Self::Ok, Self::Error>+ where+ T: Serialize + ?Sized,+ {+ value.serialize(self)+ }++ fn serialize_newtype_variant<T>(+ mut self,+ _name: &'static str,+ variant_index: u32,+ _variant: &'static str,+ value: &T,+ ) -> Result<Self::Ok, Self::Error>+ where+ T: Serialize + ?Sized,+ {+ variant_index.encode(&mut self.enc)?;+ value.serialize(self)+ }++ fn serialize_seq(mut self, len: Option<usize>) -> Result<Self::SerializeSeq, Self::Error> {+ let len = len.ok_or_else(|| SerdeEncodeError::SequenceMustHaveLength.into())?;+ len.encode(&mut self.enc)?;+ Ok(Compound { enc: self.enc })+ }++ fn serialize_tuple(self, _: usize) -> Result<Self::SerializeTuple, Self::Error> {+ Ok(self)+ }++ fn serialize_tuple_struct(+ self,+ _name: &'static str,+ _len: usize,+ ) -> Result<Self::SerializeTupleStruct, Self::Error> {+ Ok(Compound { enc: self.enc })+ }++ fn serialize_tuple_variant(+ mut self,+ _name: &'static str,+ variant_index: u32,+ _variant: &'static str,+ _len: usize,+ ) -> Result<Self::SerializeTupleVariant, Self::Error> {+ variant_index.encode(&mut self.enc)?;+ Ok(Compound { enc: self.enc })+ }++ fn serialize_map(mut self, len: Option<usize>) -> Result<Self::SerializeMap, Self::Error> {+ let len = len.ok_or_else(|| SerdeEncodeError::SequenceMustHaveLength.into())?;+ len.encode(&mut self.enc)?;+ Ok(Compound { enc: self.enc })+ }++ fn serialize_struct(+ self,+ _name: &'static str,+ _len: usize,+ ) -> Result<Self::SerializeStruct, Self::Error> {+ Ok(Compound { enc: self.enc })+ }++ fn serialize_struct_variant(+ mut self,+ _name: &'static str,+ variant_index: u32,+ _variant: &'static str,+ _len: usize,+ ) -> Result<Self::SerializeStructVariant, Self::Error> {+ variant_index.encode(&mut self.enc)?;+ Ok(Compound { enc: self.enc })+ }++ fn is_human_readable(&self) -> bool {+ false+ }+ }++ type Compound<'a, ENC> = SerdeEncoder<'a, ENC>;++ impl<'a, ENC: Encoder> SerializeSeq for Compound<'a, ENC> {+ type Ok = ();+ type Error = EncodeError;++ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>+ where+ T: Serialize + ?Sized,+ {+ value.serialize(SerdeEncoder { enc: self.enc })+ }++ fn end(self) -> Result<Self::Ok, Self::Error> {+ Ok(())+ }+ }++ impl<'a, ENC: Encoder> SerializeTuple for Compound<'a, ENC> {+ type Ok = ();+ type Error = EncodeError;++ fn serialize_element<T>(&mut self, value: &T) -> Result<(), Self::Error>+ where+ T: Serialize + ?Sized,+ {+ value.serialize(SerdeEncoder { enc: self.enc })+ }++ fn end(self) -> Result<Self::Ok, Self::Error> {+ Ok(())+ }+ }++ impl<'a, ENC: Encoder> SerializeTupleStruct for Compound<'a, ENC> {+ type Ok = ();+ type Error = EncodeError;++ fn serialize_field<T>(&mut self, value: &T) -> Result<(), Self::Error>+ where+ T: Serialize + ?Sized,+ {+ value.serialize(SerdeEncoder { enc: self.enc })+ }++ fn end(self) -> Result<Self::Ok, Self::Error> {+ Ok(())+ }+ }++ impl<'a, ENC: Encoder> SerializeTupleVariant for Compound<'a, ENC> {+ type Ok = ();+ type Error = EncodeError;++ fn serialize_field<T>(&mut self, value: &T) -> Result<(), Self::Error>+ where+ T: Serialize + ?Sized,+ {+ value.serialize(SerdeEncoder { enc: self.enc })+ }++ fn end(self) -> Result<Self::Ok, Self::Error> {+ Ok(())+ }+ }++ impl<'a, ENC: Encoder> SerializeMap for Compound<'a, ENC> {+ type Ok = ();+ type Error = EncodeError;++ fn serialize_key<T>(&mut self, key: &T) -> Result<(), Self::Error>+ where+ T: Serialize + ?Sized,+ {+ key.serialize(SerdeEncoder { enc: self.enc })+ }++ fn serialize_value<T>(&mut self, value: &T) -> Result<(), Self::Error>+ where+ T: Serialize + ?Sized,+ {+ value.serialize(SerdeEncoder { enc: self.enc })+ }++ fn end(self) -> Result<Self::Ok, Self::Error> {+ Ok(())+ }+ }++ impl<'a, ENC: Encoder> SerializeStruct for Compound<'a, ENC> {+ type Ok = ();+ type Error = EncodeError;++ fn serialize_field<T>(&mut self, _key: &'static str, value: &T) -> Result<(), Self::Error>+ where+ T: Serialize + ?Sized,+ {+ value.serialize(SerdeEncoder { enc: self.enc })+ }++ fn end(self) -> Result<Self::Ok, Self::Error> {+ Ok(())+ }+ }++ impl<'a, ENC: Encoder> SerializeStructVariant for Compound<'a, ENC> {+ type Ok = ();+ type Error = EncodeError;++ fn serialize_field<T>(&mut self, _key: &'static str, value: &T) -> Result<(), Self::Error>+ where+ T: Serialize + ?Sized,+ {+ value.serialize(SerdeEncoder { enc: self.enc })+ }++ fn end(self) -> Result<Self::Ok, Self::Error> {+ Ok(())+ }+ }+}diff --git a/crates/core/src/bloom.rs b/crates/core/src/bloom.rsindex 2bcfaed1..08277013 100644--- a/crates/core/src/bloom.rs+++ b/crates/core/src/bloom.rs@@ -31,6 +31,13 @@ impl BloomFilter {}}+ pub fn empty_from(other: &Self) -> Self {+ Self {+ bit_vec: BitVec::repeat(false, other.num_bits as usize),+ num_bits: other.num_bits,+ }+ }+fn num_bits(estimated_items: u64, fp: f64) -> u64 {((estimated_items as f64) * fp.ln() / (-8.0 * 2.0_f64.ln().powi(2))).ceil() as u64}diff --git a/crates/core/src/collector.rs b/crates/core/src/collector.rsindex 583bd5ad..86dd26ef 100644--- a/crates/core/src/collector.rs+++ b/crates/core/src/collector.rs@@ -17,7 +17,7 @@use std::{collections::HashMap, sync::Arc};use min_max_heap::MinMaxHeap;-use serde::{Deserialize, Serialize};+use tantivy::{collector::{Collector, ScoreSegmentTweaker, ScoreTweaker, SegmentCollector},DocId, SegmentOrdinal, SegmentReader,@@ -42,7 +42,16 @@ pub struct MaxDocsConsidered {pub segments: usize,}-#[derive(Clone, Copy, Debug, Serialize, Deserialize, PartialEq)]+#[derive(+ Clone,+ Copy,+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+)]pub struct Hashes {pub site: Prehashed,pub title: Prehashed,diff --git a/crates/core/src/config/mod.rs b/crates/core/src/config/mod.rsindex 273c43b2..c486f1df 100644--- a/crates/core/src/config/mod.rs+++ b/crates/core/src/config/mod.rs@@ -17,14 +17,15 @@pub mod defaults;use super::Result;+use crate::ampc::dht;use crate::distributed::member::ShardId;use crate::feed::scheduler::SplitId;-use serde::{Deserialize, Serialize};+use std::fs::File;use std::io::{self, BufRead};use std::net::SocketAddr;-#[derive(Debug, Deserialize, Clone)]+#[derive(Debug, serde::Deserialize, Clone)]pub struct IndexingLocalConfig {pub output_path: String,pub limit_warc_files: Option<usize>,@@ -44,7 +45,7 @@ pub struct IndexingLocalConfig {pub dual_encoder: Option<IndexingDualEncoderConfig>,}-#[derive(Debug, Deserialize, Clone)]+#[derive(Debug, serde::Deserialize, Clone)]pub struct IndexingDualEncoderConfig {pub model_path: String,@@ -53,7 +54,7 @@ pub struct IndexingDualEncoderConfig {pub page_centrality_rank_threshold: Option<u64>,}-#[derive(Debug, Deserialize, Clone)]+#[derive(Debug, serde::Deserialize, Clone)]pub struct WebgraphConstructConfig {pub host_graph_base_path: String,pub page_graph_base_path: String,@@ -63,7 +64,7 @@ pub struct WebgraphConstructConfig {pub batch_size: Option<usize>,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]#[serde(tag = "type")]pub enum WarcSource {HTTP(HttpConfig),@@ -130,19 +131,19 @@ impl WarcSource {}}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]pub struct LocalConfig {pub folder: String,pub names: Vec<String>,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]pub struct HttpConfig {pub base_url: String,pub warc_paths_file: String,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]pub struct S3Config {pub bucket: String,pub folder: String,@@ -151,7 +152,7 @@ pub struct S3Config {pub endpoint: String,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct CollectorConfig {#[serde(default = "defaults::Collector::site_penalty")]pub site_penalty: f64,@@ -181,7 +182,7 @@ impl Default for CollectorConfig {}}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct ApiThresholds {#[serde(default = "defaults::Api::stackoverflow")]pub stackoverflow: f64,@@ -199,14 +200,14 @@ impl Default for ApiThresholds {}}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct LLMConfig {pub api_base: String,pub model: String,pub api_key: Option<String>,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct ApiConfig {pub summarizer_path: String,pub queries_csv_path: String,@@ -239,7 +240,7 @@ pub struct ApiConfig {pub max_concurrent_searches: Option<usize>,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct SnippetConfig {#[serde(default = "defaults::Snippet::desired_num_chars")]pub desired_num_chars: usize,@@ -279,7 +280,7 @@ impl Default for SnippetConfig {}}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct SearchServerConfig {pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,@@ -299,7 +300,7 @@ pub struct SearchServerConfig {pub snippet: SnippetConfig,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct EntitySearchServerConfig {pub cluster_id: String,pub gossip_seed_nodes: Option<Vec<SocketAddr>>,@@ -308,19 +309,19 @@ pub struct EntitySearchServerConfig {pub host: SocketAddr,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct CrawlCoordinatorConfig {pub job_queue: String,pub host: SocketAddr,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct UserAgent {pub full: String,pub token: String,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct CrawlerConfig {pub num_worker_threads: usize,pub user_agent: UserAgent,@@ -354,13 +355,13 @@ pub struct CrawlerConfig {pub router_hosts: Vec<String>,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct CrawlRouterConfig {pub host: SocketAddr,pub coordinator_addrs: Vec<SocketAddr>,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]#[serde(tag = "type", content = "args", rename_all = "snake_case")]pub enum AcceleratorDevice {Cpu,@@ -368,14 +369,25 @@ pub enum AcceleratorDevice {Mps,}-#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy, Debug)]+#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Hash,+ Clone,+ Copy,+ Debug,+)]#[serde(rename_all = "snake_case")]pub enum WebgraphGranularity {Host,Page,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct WebgraphServerConfig {pub host: SocketAddr,pub graph_path: String,@@ -390,7 +402,7 @@ pub struct WebgraphServerConfig {pub max_similar_hosts: usize,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct WidgetsConfig {pub thesaurus_paths: Vec<String>,@@ -398,7 +410,7 @@ pub struct WidgetsConfig {pub calculator_fetch_currencies_exchange: bool,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct CrawlPlannerConfig {pub page_harmonic_path: String,pub host_harmonic_path: String,@@ -416,7 +428,7 @@ pub struct CrawlPlannerConfig {pub num_threads: Option<usize>,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct LiveIndexConfig {pub split_path: String,pub downloaded_db_path: String,@@ -462,7 +474,7 @@ pub struct LiveIndexConfig {pub snippet: SnippetConfig,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, Clone)]pub struct LiveIndexSchedulerConfig {pub schedule_path: String,pub feed_index_path: String,@@ -470,7 +482,7 @@ pub struct LiveIndexSchedulerConfig {pub host_graph_path: String,pub num_splits: u64,}-#[derive(Debug, Deserialize, Clone)]+#[derive(Debug, serde::Deserialize, Clone)]pub struct FeedIndexingConfig {pub output_path: String,pub warc_source: WarcSource,@@ -478,7 +490,7 @@ pub struct FeedIndexingConfig {pub skip_warc_files: Option<usize>,}-#[derive(Debug, Deserialize, Clone)]+#[derive(Debug, serde::Deserialize, Clone)]pub struct WebSpellConfig {pub output_path: String,pub warc_source: WarcSource,@@ -513,3 +525,34 @@ impl Default for CorrectionConfig {}}}++#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]+pub struct GossipConfig {+ pub cluster_id: String,+ pub seed_nodes: Option<Vec<SocketAddr>>,+ pub addr: SocketAddr,+}++#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]+pub struct DhtConfig {+ pub node_id: dht::NodeId,+ pub host: SocketAddr,+ pub shard: ShardId,+ pub seed_node: Option<SocketAddr>,+ pub gossip: GossipConfig,+}++#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]+pub struct HarmonicCoordinatorConfig {+ pub gossip: GossipConfig,+ pub host: SocketAddr,+ pub output_path: String,+}++#[derive(Clone, Debug, serde::Deserialize, serde::Serialize)]+pub struct HarmonicWorkerConfig {+ pub gossip: GossipConfig,+ pub shard: ShardId,+ pub graph_path: String,+ pub host: SocketAddr,+}diff --git a/crates/core/src/crawler/file_queue.rs b/crates/core/src/crawler/file_queue.rsindex 5b0b07a9..c3159883 100644--- a/crates/core/src/crawler/file_queue.rs+++ b/crates/core/src/crawler/file_queue.rs@@ -16,6 +16,7 @@use anyhow::Result;use memmap2::Mmap;+use postcard::experimental::max_size::MaxSize;use std::{fs::File,io::{BufWriter, Read, Seek, Write},@@ -25,7 +26,7 @@ use std::{const POINTER_KEY: &str = "pointer";const DATA_KEY: &str = "data";-#[derive(Clone, Copy, serde::Serialize, serde::Deserialize)]+#[derive(Debug, Clone, Copy, MaxSize, serde::Serialize, serde::Deserialize)]struct Header {body_size: usize,}@@ -38,7 +39,7 @@ pub struct FileQueueWriter<T> {impl<T> FileQueueWriter<T>where- T: serde::Serialize + serde::de::DeserializeOwned,+ T: bincode::Encode + bincode::Decode,{pub fn new<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {if !path.as_ref().exists() {@@ -60,12 +61,16 @@ where}pub fn push(&mut self, item: T) -> Result<()> {- let body = bincode::serialize(&item)?;+ let body = bincode::encode_to_vec(&item, bincode::config::standard())?;let header = Header {body_size: body.len(),};- let header_bytes = bincode::serialize(&header)?;+ let mut header_bytes = postcard::to_allocvec(&header).unwrap();++ if header_bytes.len() < Header::POSTCARD_MAX_SIZE {+ header_bytes.resize(Header::POSTCARD_MAX_SIZE, 0);+ }self.writer.write_all(&header_bytes)?;self.writer.write_all(&body)?;@@ -140,7 +145,7 @@ pub struct FileQueue<T> {impl<T> FileQueue<T>where- T: serde::Serialize + serde::de::DeserializeOwned,+ T: bincode::Encode + bincode::Decode,{pub fn open<P: AsRef<std::path::Path>>(path: P) -> Result<Self> {if !path.as_ref().exists() {@@ -164,13 +169,15 @@ wherereturn Ok(None);}- let header_size = std::mem::size_of::<Header>();- let header: Header =- bincode::deserialize(&self.file[cur_pointer..cur_pointer + header_size])?;+ let header_size = Header::POSTCARD_MAX_SIZE;++ let header_bytes = &self.file[cur_pointer..cur_pointer + header_size];++ let header: Header = postcard::from_bytes(header_bytes).unwrap();let body =&self.file[cur_pointer + header_size..cur_pointer + header_size + header.body_size];- let item = bincode::deserialize(body)?;+ let (item, _) = bincode::decode_from_slice(body, bincode::config::standard())?;self.pointer.set(cur_pointer + header_size + header.body_size)?;diff --git a/crates/core/src/crawler/mod.rs b/crates/core/src/crawler/mod.rsindex 9edefef8..186c584e 100644--- a/crates/core/src/crawler/mod.rs+++ b/crates/core/src/crawler/mod.rs@@ -16,7 +16,7 @@use std::{collections::VecDeque, future::Future, sync::Arc, time::Duration};-use hashbrown::HashMap;+type HashMap<K, V> = std::collections::HashMap<K, V, ahash::RandomState>;use url::Url;@@ -65,11 +65,31 @@ pub enum Error {type Result<T, E = anyhow::Error> = std::result::Result<T, E>;#[derive(- Clone, Debug, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,+ Clone,+ Debug,+ PartialEq,+ Eq,+ Hash,+ PartialOrd,+ Ord,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,)]struct Site(String);-#[derive(Clone, Debug, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Clone,+ Debug,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Domain(String);impl From<&Url> for Domain {@@ -96,8 +116,9 @@ impl Domain {}}-#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]pub struct WeightedUrl {+ #[bincode(with_serde)]pub url: Url,pub weight: f64,}@@ -118,22 +139,44 @@ impl std::hash::Hash for WeightedUrl {/// All urls in a job must be from the same domain and only one job per domain./// at a time. This ensures that we stay polite when crawling.-#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]pub struct Job {pub domain: Domain,pub urls: VecDeque<WeightedUrl>,pub wandering_urls: u64,}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub enum UrlResponse {- Success { url: Url },- Failed { url: Url, status_code: Option<u16> },- Redirected { url: Url, new_url: Url },+ Success {+ #[bincode(with_serde)]+ url: Url,+ },+ Failed {+ #[bincode(with_serde)]+ url: Url,+ status_code: Option<u16>,+ },+ Redirected {+ #[bincode(with_serde)]+ url: Url,+ #[bincode(with_serde)]+ new_url: Url,+ },}#[derive(- Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,+ Debug,+ Clone,+ PartialEq,+ Eq,+ Hash,+ PartialOrd,+ Ord,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,)]pub struct UrlString(String);@@ -156,18 +199,18 @@ impl TryFrom<&UrlString> for Url {}}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct UrlToInsert {pub url: UrlString,pub weight: f64,}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct DiscoveredUrls {pub urls: HashMap<Domain, Vec<UrlToInsert>>,}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct DomainCrawled {pub domain: Domain,pub budget_used: f64,diff --git a/crates/core/src/crawler/planner.rs b/crates/core/src/crawler/planner.rsindex 94416ce6..9a26939f 100644--- a/crates/core/src/crawler/planner.rs+++ b/crates/core/src/crawler/planner.rs@@ -30,7 +30,7 @@ use std::{use url::Url;use crate::crawler::WeightedUrl;-use crate::webgraph::centrality::{top_hosts, TopHosts};+use crate::webgraph::centrality::{top_nodes, TopNodes};use crate::SortableFloat;use crate::{config::CrawlPlannerConfig,@@ -311,10 +311,13 @@ impl CrawlPlanner {let queue_path = output.as_ref().join("job_queue");std::fs::create_dir_all(&queue_path)?;- let hosts = top_hosts(+ let hosts = top_nodes(&self.host_centrality,- TopHosts::Fraction(self.config.top_host_fraction),- );+ TopNodes::Fraction(self.config.top_host_fraction),+ )+ .into_iter()+ .map(|(host, _)| host)+ .collect::<Vec<_>>();tracing::info!("found {} hosts", hosts.len());let grouped_hosts = self.group_domain(&hosts);diff --git a/crates/core/src/distributed/member.rs b/crates/core/src/distributed/member.rsindex ce1a32d9..87c074f1 100644--- a/crates/core/src/distributed/member.rs+++ b/crates/core/src/distributed/member.rs@@ -16,11 +16,22 @@use std::net::SocketAddr;-use serde::{Deserialize, Serialize};-use crate::config::WebgraphGranularity;-#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Copy, Debug, PartialOrd, Ord)]+#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Hash,+ Clone,+ Copy,+ Debug,+ PartialOrd,+ Ord,+)]pub struct ShardId(u64);impl ShardId {@@ -35,7 +46,23 @@ impl From<u64> for ShardId {}}-#[derive(Serialize, Deserialize, PartialEq, Eq, Hash, Clone, Debug)]+impl From<ShardId> for u64 {+ fn from(id: ShardId) -> u64 {+ id.0+ }+}++#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Hash,+ Clone,+ Debug,+)]pub enum Service {Searcher {host: SocketAddr,@@ -59,6 +86,13 @@ pub enum Service {host: SocketAddr,shard: ShardId,},+ HarmonicWorker {+ host: SocketAddr,+ shard: ShardId,+ },+ HarmonicCoordinator {+ host: SocketAddr,+ },}#[derive(PartialEq, Eq, Hash, Clone, Debug)]diff --git a/crates/core/src/distributed/sonic/mod.rs b/crates/core/src/distributed/sonic/mod.rsindex 203d5e4f..3be9e8a0 100644--- a/crates/core/src/distributed/sonic/mod.rs+++ b/crates/core/src/distributed/sonic/mod.rs@@ -19,7 +19,6 @@ pub mod service;use std::{marker::PhantomData, time::Duration};-use serde::{de::DeserializeOwned, Serialize};use tokio::{io::{AsyncReadExt, AsyncWriteExt},net::{TcpListener, TcpStream, ToSocketAddrs},@@ -34,9 +33,6 @@ pub enum Error {#[error("Got an IO error")]IO(#[from] std::io::Error),- #[error("Error while serializing/deserializing to/from bytes")]- Serialization(#[from] bincode::Error),-#[error("Failed to connect to peer: connection timeout")]ConnectionTimeout,@@ -63,9 +59,13 @@ pub struct Connection<Req, Res> {impl<Req, Res> Connection<Req, Res>where- Req: Serialize,- Res: DeserializeOwned,+ Req: bincode::Encode,+ Res: bincode::Decode,{+ pub async fn connect(server: impl ToSocketAddrs) -> Result<Self> {+ Self::create(server).await+ }+pub async fn create(server: impl ToSocketAddrs) -> Result<Self> {Self::create_with_timeout(server, Duration::from_secs(30)).await}@@ -110,7 +110,7 @@ where}async fn send_without_timeout(mut self, request: &Req) -> Result<Res> {- let bytes = bincode::serialize(&request).unwrap();+ let bytes = bincode::encode_to_vec(request, bincode::config::standard()).unwrap();// disable linger to avoid TIME_WAIT.// should be safe since the connection is closed from the client side.@@ -142,7 +142,9 @@ whereself.stream.shutdown().await?;tracing::debug!("deserializing {:?}", std::any::type_name::<(Req, Res)>());- Ok(bincode::deserialize(&buf).unwrap())+ let (res, _) = bincode::decode_from_slice(&buf, bincode::config::standard()).unwrap();++ Ok(res)}pub async fn send(self, request: &Req) -> Result<Res> {@@ -171,7 +173,7 @@ pub struct Server<Req, Res> {impl<Req, Res> Server<Req, Res>where- Req: DeserializeOwned,+ Req: bincode::Decode,{pub async fn bind(addr: impl ToSocketAddrs) -> Result<Self> {let listener = TcpListener::bind(addr).await?;@@ -197,11 +199,11 @@ wherestream.read_exact(&mut buf).await?;- let body = Some(bincode::deserialize(&buf).unwrap());+ let (body, _) = bincode::decode_from_slice(&buf, bincode::config::standard()).unwrap();Ok(Request {stream,- body,+ body: Some(body),marker: PhantomData,})}@@ -224,10 +226,10 @@ pub struct Request<Req, Res> {impl<Req, Res> Request<Req, Res>where- Res: Serialize,+ Res: bincode::Encode,{async fn respond_without_timeout(mut self, response: Res) -> Result<()> {- let bytes = bincode::serialize(&response).unwrap();+ let bytes = bincode::encode_to_vec(&response, bincode::config::standard()).unwrap();let header = Header {body_size: bytes.len(),};@@ -269,15 +271,14 @@ mod tests {use proptest::prelude::*;use proptest_derive::Arbitrary;- use serde::Deserialize;use crate::free_socket_addr;use super::*;fn fixture<- Req: Serialize + DeserializeOwned + Send + 'static,- Res: Serialize + DeserializeOwned + Send + 'static,+ Req: bincode::Encode + bincode::Decode + Send + 'static,+ Res: bincode::Encode + bincode::Decode + Send + 'static,A: Send + 'static,B: Send + 'static,X: Future<Output = Result<A, TestCaseError>> + Send,@@ -306,7 +307,7 @@ mod tests {})}- #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Arbitrary)]+ #[derive(Debug, Clone, bincode::Encode, bincode::Decode, PartialEq, Arbitrary)]struct Message {text: String,other: HashMap<String, f32>,diff --git a/crates/core/src/distributed/sonic/replication.rs b/crates/core/src/distributed/sonic/replication.rsindex 0ca6e34d..7e07f26a 100644--- a/crates/core/src/distributed/sonic/replication.rs+++ b/crates/core/src/distributed/sonic/replication.rs@@ -21,12 +21,43 @@ use super::Result;use crate::distributed::{retry_strategy::ExponentialBackoff, sonic};use std::{net::SocketAddr, time::Duration};-#[derive(Debug)]-pub struct RemoteClient<S: sonic::service::Service> {+#[derive(Debug, serde::Serialize, serde::Deserialize)]+pub struct RemoteClient<S> {addr: SocketAddr,_phantom: std::marker::PhantomData<S>,}+impl<S> bincode::Encode for RemoteClient<S> {+ fn encode<E: bincode::enc::Encoder>(+ &self,+ encoder: &mut E,+ ) -> Result<(), bincode::error::EncodeError> {+ self.addr.encode(encoder)+ }+}++impl<S> bincode::Decode for RemoteClient<S> {+ fn decode<D: bincode::de::Decoder>(+ decoder: &mut D,+ ) -> Result<Self, bincode::error::DecodeError> {+ Ok(Self {+ addr: SocketAddr::decode(decoder)?,+ _phantom: std::marker::PhantomData,+ })+ }+}++impl<'de, S> bincode::BorrowDecode<'de> for RemoteClient<S> {+ fn borrow_decode<D: bincode::de::BorrowDecoder<'de>>(+ decoder: &mut D,+ ) -> Result<Self, bincode::error::DecodeError> {+ Ok(Self {+ addr: SocketAddr::borrow_decode(decoder)?,+ _phantom: std::marker::PhantomData,+ })+ }+}+impl<S> Clone for RemoteClient<S>whereS: sonic::service::Service,@@ -50,6 +81,10 @@ where_phantom: std::marker::PhantomData,}}++ pub fn addr(&self) -> SocketAddr {+ self.addr+ }}impl<S> RemoteClient<S>diff --git a/crates/core/src/distributed/sonic/service.rs b/crates/core/src/distributed/sonic/service.rsindex 6efea7f9..e7ac74fe 100644--- a/crates/core/src/distributed/sonic/service.rs+++ b/crates/core/src/distributed/sonic/service.rs@@ -21,9 +21,9 @@ use tokio::net::ToSocketAddrs;use super::Result;pub trait Service: Sized + Send + Sync + 'static {- type Request: serde::de::DeserializeOwned + Send + Sync;- type RequestRef<'a>: serde::Serialize + Send + Sync;- type Response: serde::Serialize + serde::de::DeserializeOwned + Send + Sync;+ type Request: bincode::Decode + Send + Sync;+ type RequestRef<'a>: bincode::Encode + Send + Sync;+ type Response: bincode::Encode + bincode::Decode + Send + Sync;fn handle(req: Self::Request,@@ -128,7 +128,6 @@ impl<'a, S: Service> Connection<'a, S> {}}-#[macro_export]macro_rules! sonic_service {($service:ident, [$($req:ident),*$(,)?]) => {mod service_impl__ {@@ -138,15 +137,15 @@ macro_rules! sonic_service {use $crate::distributed::sonic;- #[derive(Debug, Clone, ::serde::Deserialize)]+ #[derive(Debug, Clone, ::bincode::Decode)]pub enum Request {$($req($req),)*}- #[derive(Debug, Clone, ::serde::Serialize)]+ #[derive(Debug, Clone, ::bincode::Encode)]pub enum RequestRef<'a> {$($req(&'a $req),)*}- #[derive(::serde::Serialize, ::serde::Deserialize)]+ #[derive(::bincode::Encode, ::bincode::Decode)]pub enum Response {$($req(<$req as sonic::service::Message<$service>>::Response),)*}@@ -193,6 +192,8 @@ macro_rules! sonic_service {};}+pub(crate) use sonic_service;+#[cfg(test)]mod tests {use proptest::prelude::*;@@ -260,7 +261,6 @@ mod tests {use std::sync::atomic::AtomicI32;use proptest_derive::Arbitrary;- use serde::{Deserialize, Serialize};use super::super::Message;@@ -270,11 +270,21 @@ mod tests {sonic_service!(CounterService, [Change, Reset]);- #[derive(Debug, Clone, Serialize, Deserialize, Arbitrary)]+ #[derive(+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Arbitrary,+ )]pub struct Change {pub amount: i32,}- #[derive(Debug, Clone, Serialize, Deserialize)]+ #[derive(+ Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,+ )]pub struct Reset;impl Message<CounterService> for Change {diff --git a/crates/core/src/entity_index/entity.rs b/crates/core/src/entity_index/entity.rsindex ab1b4abf..5ae39ee1 100644--- a/crates/core/src/entity_index/entity.rs+++ b/crates/core/src/entity_index/entity.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use itertools::Itertools;-use serde::{Deserialize, Serialize};+use utoipa::ToSchema;#[derive(Debug)]@@ -28,7 +28,17 @@ pub struct Entity {pub image: Option<String>,}-#[derive(Debug, Default, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[derive(+ Debug,+ Default,+ Clone,+ PartialEq,+ Eq,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Span {pub text: String,pub links: Vec<Link>,@@ -87,21 +97,49 @@ impl Span {}}-#[derive(Debug, PartialEq, Eq, Serialize, Deserialize, Clone)]+#[derive(+ Debug,+ PartialEq,+ Eq,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Clone,+)]pub struct Link {pub start: usize,pub end: usize,pub target: String,}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, ToSchema)]+#[derive(+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ ToSchema,+)]#[serde(rename_all = "camelCase", tag = "kind")]pub enum EntitySnippetFragment {Normal { text: String },Link { text: String, href: String },}-#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, ToSchema)]+#[derive(+ Default,+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct EntitySnippet {pub fragments: Vec<EntitySnippetFragment>,diff --git a/crates/core/src/entity_index/mod.rs b/crates/core/src/entity_index/mod.rsindex 563c24a3..184e51db 100644--- a/crates/core/src/entity_index/mod.rs+++ b/crates/core/src/entity_index/mod.rs@@ -17,7 +17,7 @@use std::{collections::HashSet, fs, path::Path, sync::Arc};use base64::{prelude::BASE64_STANDARD as BASE64_ENGINE, Engine};-use serde::{Deserialize, Serialize};+use tantivy::{collector::TopDocs,query::{BooleanQuery, BoostQuery, MoreLikeThisQuery, Occur, QueryClone, TermQuery},@@ -89,11 +89,11 @@ fn entity_to_tantivy(entity: Entity, schema: &tantivy::schema::Schema) -> Tantiv);doc.add_bytes(schema.get_field("info").unwrap(),- bincode::serialize(&entity.info).unwrap(),+ bincode::encode_to_vec(&entity.info, bincode::config::standard()).unwrap(),);doc.add_bytes(schema.get_field("links").unwrap(),- bincode::serialize(&entity.page_abstract.links).unwrap(),+ bincode::encode_to_vec(&entity.page_abstract.links, bincode::config::standard()).unwrap(),);let has_image = if entity.image.is_some() {"true"@@ -110,7 +110,7 @@ fn entity_to_tantivy(entity: Entity, schema: &tantivy::schema::Schema) -> Tantivdoc}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]pub struct StoredEntity {pub title: String,pub entity_abstract: String,@@ -120,7 +120,7 @@ pub struct StoredEntity {pub links: Vec<Link>,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]pub struct EntityMatch {pub entity: StoredEntity,pub score: f32,@@ -340,15 +340,18 @@ impl EntityIndex {.unwrap();let info = if decode_info {- bincode::deserialize(+ let (info, _) = bincode::decode_from_slice(doc.get_first(info).and_then(|val| match val {tantivy::schema::OwnedValue::Bytes(bytes) => Some(bytes),_ => None,}).unwrap(),+ bincode::config::standard(),)- .unwrap()+ .unwrap();++ info} else {Vec::new()};@@ -382,15 +385,18 @@ impl EntityIndex {};let links: Vec<Link> = if get_links {- bincode::deserialize(+ let (links, _) = bincode::decode_from_slice(doc.get_first(links).and_then(|val| match val {tantivy::schema::OwnedValue::Bytes(bytes) => Some(bytes),_ => None,}).unwrap(),+ bincode::config::standard(),)- .unwrap()+ .unwrap();++ links} else {Vec::new()};diff --git a/crates/core/src/entrypoint/ampc/dht.rs b/crates/core/src/entrypoint/ampc/dht.rsnew file mode 100644index 00000000..74afb2c7--- /dev/null+++ b/crates/core/src/entrypoint/ampc/dht.rs@@ -0,0 +1,165 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++use std::{collections::BTreeMap, net::SocketAddr, sync::Arc};++use anyhow::bail;+use openraft::error::InitializeError;+use tracing::info;++use crate::{+ ampc::dht::{self, BasicNode, ShardId},+ config::{DhtConfig, GossipConfig},+ distributed::{+ cluster::Cluster,+ member::{Member, Service},+ },+ Result,+};++pub struct Config {+ pub node_id: dht::NodeId,+ pub host: SocketAddr,+ pub shard: ShardId,+ pub seed_node: Option<SocketAddr>,+ pub gossip: Option<GossipConfig>,+}++impl From<DhtConfig> for Config {+ fn from(config: DhtConfig) -> Self {+ Self {+ node_id: config.node_id,+ host: config.host,+ shard: config.shard,+ seed_node: config.seed_node,+ gossip: Some(config.gossip),+ }+ }+}++pub async fn run<C: Into<Config>>(config: C) -> Result<()> {+ let config: Config = config.into();++ let raft_config = openraft::Config::default();+ let raft_config = Arc::new(raft_config.validate()?);++ let log_store = dht::log_store::LogStore::<dht::TypeConfig>::default();+ let state_machine_store = Arc::new(dht::store::StateMachineStore::default());++ let network = dht::network::Network;++ let raft = openraft::Raft::new(+ config.node_id,+ raft_config,+ network,+ log_store,+ state_machine_store.clone(),+ )+ .await?;++ let server = dht::Server::new(raft.clone(), state_machine_store)+ .bind(config.host)+ .await?;++ match config.seed_node {+ Some(seed) => {+ let client = dht::RaftClient::new(seed).await?;+ let metrics = client.metrics().await?;++ if metrics+ .membership_config+ .nodes()+ .any(|(id, node)| *id == config.node_id || node.addr == config.host.to_string())+ {+ bail!("Already member of cluster. It is currently not safe for a node to rejoin a cluster it was previously a member of.");+ }++ client.join(config.node_id, config.host).await?;++ info!("Joined cluster with node_id: {}", config.node_id);+ }+ None => {+ let members: BTreeMap<u64, _> =+ BTreeMap::from([(config.node_id, BasicNode::new(config.host))]);++ if let Err(e) = raft.initialize(members.clone()).await {+ match e {+ openraft::error::RaftError::APIError(e) => match e {+ InitializeError::NotAllowed(_) => {}+ InitializeError::NotInMembers(_) => bail!(e),+ },+ openraft::error::RaftError::Fatal(_) => bail!(e),+ }+ }++ info!("Initialized cluster with node_id: {}", config.node_id);+ }+ }++ // dropping the handle leaves the cluster+ let _cluster_handle = match config.gossip {+ Some(gossip) => Some(+ Cluster::join(+ Member {+ id: gossip.cluster_id,+ service: Service::Dht {+ host: config.host,+ shard: config.shard,+ },+ },+ gossip.addr,+ gossip.seed_nodes.unwrap_or_default(),+ )+ .await?,+ ),+ None => None,+ };++ loop {+ server.accept().await?;+ }+}++#[cfg(test)]+pub mod tests {+ use super::*;+ use crate::free_socket_addr;++ pub fn setup() -> (ShardId, SocketAddr) {+ let (tx, rx) = crossbeam_channel::unbounded();+ std::thread::spawn(move || {+ let rt = tokio::runtime::Runtime::new().unwrap();+ let shard = ShardId::new(1);++ rt.block_on(async {+ let addr = free_socket_addr();+ tx.send((shard, addr)).unwrap();++ run(Config {+ node_id: 1,+ host: addr,+ seed_node: None,+ shard,+ gossip: None,+ })+ .await+ .unwrap();+ })+ });++ rx.recv().unwrap()+ }+}diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rsnew file mode 100644index 00000000..b39d2618--- /dev/null+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/coordinator.rs@@ -0,0 +1,212 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use crate::{+ ampc::{dht::ShardId, prelude::*, Coordinator, DhtConn},+ config::HarmonicCoordinatorConfig,+ distributed::member::Member,+ webgraph::centrality::{store_csv, store_harmonic, top_nodes, TopNodes},+ Result,+};+use std::{collections::BTreeMap, net::SocketAddr, path::Path};++use crate::{+ ampc::DefaultDhtTable,+ distributed::{cluster::Cluster, member::Service},+};++use super::{CentralityJob, CentralityMapper, CentralityTables, Meta, RemoteCentralityWorker};++pub struct CentralitySetup {+ dht: DhtConn<CentralityTables>,+}++impl CentralitySetup {+ pub async fn new(cluster: &Cluster) -> Self {+ let members: Vec<_> = cluster+ .members()+ .await+ .into_iter()+ .filter_map(|member| {+ if let Service::Dht { host, shard } = member.service {+ Some((shard, host))+ } else {+ None+ }+ })+ .collect();++ Self::new_for_dht_members(&members)+ }++ pub fn new_for_dht_members(members: &[(ShardId, SocketAddr)]) -> Self {+ let initial = CentralityTables {+ counters: DefaultDhtTable::new(members, "counters"),+ meta: DefaultDhtTable::new(members, "meta"),+ centrality: DefaultDhtTable::new(members, "centrality"),+ };++ let dht = DhtConn::new(initial);++ Self { dht }+ }+}++impl Setup for CentralitySetup {+ type DhtTables = CentralityTables;++ fn init_dht(&self) -> DhtConn<Self::DhtTables> {+ self.dht.clone()+ }++ fn setup_round(&self, dht: &Self::DhtTables) {+ dht.meta.set(+ (),+ Meta {+ round_had_changes: false,+ },+ );+ }++ fn setup_first_round(&self, dht: &Self::DhtTables) {+ dht.meta.set(+ (),+ Meta {+ round_had_changes: true, // force first round to run+ },+ );+ }+}++pub struct CentralityFinish;++impl Finisher for CentralityFinish {+ type Job = CentralityJob;++ fn is_finished(&self, dht: &CentralityTables) -> bool {+ !dht.meta.get(()).unwrap().round_had_changes+ }+}++pub fn build(+ dht: &[(ShardId, SocketAddr)],+ workers: Vec<RemoteCentralityWorker>,+) -> Coordinator<CentralityJob> {+ let setup = CentralitySetup::new_for_dht_members(dht);++ Coordinator::new(setup, workers)+ .with_mapper(CentralityMapper::SetupCounters)+ .with_mapper(CentralityMapper::Cardinalities)+ .with_mapper(CentralityMapper::Centralities)+}++struct ClusterInfo {+ // dropping the handle will leave the cluster+ _handle: Cluster,+ dht: Vec<(ShardId, SocketAddr)>,+ workers: Vec<RemoteCentralityWorker>,+}++async fn setup_gossip(config: HarmonicCoordinatorConfig) -> Result<ClusterInfo> {+ let handle = Cluster::join(+ Member {+ id: config.gossip.cluster_id,+ service: Service::HarmonicCoordinator { host: config.host },+ },+ config.gossip.addr,+ config.gossip.seed_nodes.unwrap_or_default(),+ )+ .await?;++ tokio::time::sleep(tokio::time::Duration::from_secs(5)).await;++ let members = handle.members().await;++ let dht = members+ .iter()+ .filter_map(|member| {+ if let Service::Dht { host, shard } = member.service {+ Some((shard, host))+ } else {+ None+ }+ })+ .collect();++ let workers = members+ .iter()+ .filter_map(|member| {+ if let Service::HarmonicWorker { host, shard } = member.service {+ Some(RemoteCentralityWorker::new(shard, host))+ } else {+ None+ }+ })+ .collect();++ Ok(ClusterInfo {+ _handle: handle,+ dht,+ workers,+ })+}++pub fn run(config: HarmonicCoordinatorConfig) -> Result<()> {+ let tokio_conf = config.clone();+ let cluster = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()?+ .block_on(setup_gossip(tokio_conf))?;++ let jobs = cluster+ .workers+ .iter()+ .map(|worker| CentralityJob {+ shard: worker.shard(),+ })+ .collect();++ let coordinator = build(&cluster.dht, cluster.workers.clone());+ let res = coordinator.run(jobs, CentralityFinish)?;++ let num_nodes = cluster.workers.iter().map(|w| w.num_nodes()).sum::<u64>();+ let output_path = Path::new(&config.output_path);++ let store = store_harmonic(+ res.centrality+ .iter()+ .map(|(n, c)| (n, f64::from(c) / (num_nodes - 1) as f64)),+ output_path,+ );++ let top_nodes = top_nodes(&store, TopNodes::Top(1_000_000));++ let ids = top_nodes.iter().map(|(id, _)| *id).collect::<Vec<_>>();+ let id2node: BTreeMap<_, _> = cluster+ .workers+ .iter()+ .flat_map(|w| w.batch_id2node(ids.clone()))+ .collect();++ let top_nodes = top_nodes+ .iter()+ .map(|(id, c)| (id2node[id].clone(), *c))+ .collect::<Vec<_>>();++ store_csv(top_nodes, output_path.join("harmonic.csv"));++ Ok(())+}diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rsnew file mode 100644index 00000000..5d8cdacc--- /dev/null+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mapper.rs@@ -0,0 +1,323 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use std::{+ collections::BTreeMap,+ sync::{Arc, Mutex},+};++use crate::{+ ampc::{+ dht::{HyperLogLog64Upsert, UpsertAction},+ prelude::*,+ DhtConn,+ },+ bloom::BloomFilter,+ hyperloglog::HyperLogLog,+ webgraph,+};++use super::{CentralityJob, CentralityTables, CentralityWorker, Meta};++const OPS_BATCH_PER_SHARD: u64 = 4096;++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub enum CentralityMapper {+ SetupCounters,+ Cardinalities,+ Centralities,+}++impl CentralityMapper {+ /// get old values from prev dht using edge.from where edge.from in changed_nodes+ fn get_old_counters(+ batch: &[webgraph::Edge<()>],+ dht: &DhtConn<CentralityTables>,+ ) -> BTreeMap<webgraph::NodeID, HyperLogLog<64>> {+ let nodes: Vec<_> = batch.iter().map(|edge| edge.from).collect();++ if nodes.is_empty() {+ return BTreeMap::new();+ }++ dht.prev().counters.batch_get(nodes).into_iter().collect()+ }++ fn setup_counters(nodes: &[webgraph::NodeID], dht: &DhtConn<CentralityTables>) {+ dht.prev().counters.batch_set(+ nodes+ .iter()+ .map(|node| {+ let mut hll = HyperLogLog::default();+ hll.add(node.as_u64());+ (*node, hll)+ })+ .collect(),+ );++ dht.next().counters.batch_set(+ nodes+ .iter()+ .map(|node| {+ let mut hll = HyperLogLog::default();+ hll.add(node.as_u64());+ (*node, hll)+ })+ .collect(),+ );+ }++ /// upsert old edge.from `old_counters` into edge.to in dht.next,+ /// thereby updating their hyperloglog counters+ fn update_counters(+ batch: &[webgraph::Edge<()>],+ dht: &DhtConn<CentralityTables>,+ ) -> Vec<(webgraph::NodeID, UpsertAction)> {+ let old_counters = Self::get_old_counters(batch, dht);++ let updates: Vec<_> = batch+ .iter()+ .map(|edge| {+ let mut counter = old_counters.get(&edge.from).cloned().unwrap_or_default();+ counter.add(edge.from.as_u64());+ (edge.to, counter)+ })+ .collect();++ if updates.is_empty() {+ return vec![];+ }++ dht.next()+ .counters+ .batch_upsert(HyperLogLog64Upsert, updates)+ }++ /// update new bloom filter with the nodes that changed+ fn update_changed_nodes(+ changes: &[(webgraph::NodeID, UpsertAction)],+ new_changed_nodes: &Mutex<BloomFilter>,+ ) {+ let mut new_changed_nodes = new_changed_nodes.lock().unwrap();++ for (node, upsert_res) in changes {+ if let UpsertAction::Merged = upsert_res {+ new_changed_nodes.insert(node.as_u64());+ }+ }+ }++ fn update_dht(+ batch: &[webgraph::Edge<()>],+ new_changed_nodes: &Mutex<BloomFilter>,+ dht: &DhtConn<CentralityTables>,+ ) {+ if batch.is_empty() {+ return;+ }++ let changes = Self::update_counters(batch, dht);+ Self::update_changed_nodes(&changes, new_changed_nodes);++ // if any nodes changed, indicate in dht that we aren't finished yet+ if changes.iter().any(|(_, upsert_res)| match upsert_res {+ UpsertAction::Merged => true,+ UpsertAction::NoChange => false,+ UpsertAction::Inserted => true,+ }) {+ dht.next().meta.set(+ (),+ Meta {+ round_had_changes: true,+ },+ )+ }+ }++ fn update_centralities(+ nodes: &[webgraph::NodeID],+ round: u64,+ dht: &DhtConn<CentralityTables>,+ ) {+ if nodes.is_empty() {+ return;+ }++ let old_counters: BTreeMap<_, _> = dht+ .prev()+ .counters+ .batch_get(nodes.to_vec())+ .into_iter()+ .collect();+ let new_counters: BTreeMap<_, _> = dht+ .next()+ .counters+ .batch_get(nodes.to_vec())+ .into_iter()+ .collect();++ let mut new_values = BTreeMap::new();+ let old_values: BTreeMap<_, _> = dht+ .prev()+ .centrality+ .batch_get(nodes.to_vec())+ .into_iter()+ .collect();++ for node in nodes {+ if let (Some(old_size), Some(new_size)) = (+ old_counters.get(node).map(|s| s.size() as u64),+ new_counters.get(node).map(|s| s.size() as u64),+ ) {+ let d = new_size.saturating_sub(old_size);++ if d == 0 {+ continue;+ }++ new_values.insert(+ *node,+ old_values.get(node).copied().unwrap_or(0.0.into())+ + (d as f64 / (round + 1) as f64),+ );+ }+ }++ dht.next()+ .centrality+ .batch_set(new_values.into_iter().collect());+ }++ fn map_setup_counters(worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {+ // shards are the same for both prev and next+ let num_shards = dht.prev().num_shards();+ let batch_size = (num_shards * OPS_BATCH_PER_SHARD) as usize;++ let pool = rayon::ThreadPoolBuilder::new().build().unwrap();++ if worker.round() == 0 {+ pool.scope(|s| {+ let mut batch = Vec::with_capacity(batch_size);+ let mut changed_nodes = worker.changed_nodes().lock().unwrap();++ for node in worker.graph().nodes() {+ changed_nodes.insert(node.as_u64());+ batch.push(node);+ if batch.len() >= batch_size {+ let update_batch = batch.clone();+ s.spawn(move |_| Self::setup_counters(&update_batch, dht));++ batch.clear();+ }+ }++ if !batch.is_empty() {+ let update_batch = batch.clone();+ s.spawn(move |_| Self::setup_counters(&update_batch, dht));+ }+ });+ }+ }++ fn map_cardinalities(worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {+ // shards are the same for both prev and next+ let num_shards = dht.prev().num_shards();+ let batch_size = (num_shards * OPS_BATCH_PER_SHARD) as usize;++ let new_changed_nodes = Arc::new(Mutex::new(BloomFilter::empty_from(+ &worker.changed_nodes().lock().unwrap(),+ )));++ let pool = rayon::ThreadPoolBuilder::new().build().unwrap();++ pool.scope(|s| {+ let mut batch = Vec::with_capacity(batch_size);+ let changed_nodes = worker.changed_nodes().lock().unwrap();++ for edge in worker+ .graph()+ .edges()+ .filter(|e| changed_nodes.contains(e.from.as_u64()))+ {+ batch.push(edge);+ if batch.len() >= batch_size {+ let new_changed_nodes = Arc::clone(&new_changed_nodes);+ let update_batch = batch.clone();++ s.spawn(move |_| Self::update_dht(&update_batch, &new_changed_nodes, dht));++ batch.clear();+ }+ }++ if !batch.is_empty() {+ let new_changed_nodes = Arc::clone(&new_changed_nodes);+ let update_batch = batch.clone();++ s.spawn(move |_| Self::update_dht(&update_batch, &new_changed_nodes, dht));+ }+ });+ *worker.changed_nodes().lock().unwrap() = new_changed_nodes.lock().unwrap().clone();+ }++ fn map_centralities(worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {+ if !dht.prev().meta.get(()).unwrap().round_had_changes {+ return;+ }++ // shards are the same for both prev and next+ let num_shards = dht.prev().num_shards();+ let batch_size = (num_shards * OPS_BATCH_PER_SHARD) as usize;++ let pool = rayon::ThreadPoolBuilder::new().build().unwrap();+ let round = worker.inc_round();++ // count cardinality of hyperloglogs in dht.next and update count after all mappers are done+ pool.scope(|s| {+ let mut batch = Vec::with_capacity(batch_size);+ let changed_nodes = worker.changed_nodes().lock().unwrap();+ for node in worker+ .graph()+ .nodes()+ .filter(|n| changed_nodes.contains(n.as_u64()))+ {+ batch.push(node);+ if batch.len() >= batch_size {+ let update_batch = batch.clone();+ s.spawn(move |_| Self::update_centralities(&update_batch, round, dht));++ batch.clear();+ }+ }++ if !batch.is_empty() {+ s.spawn(move |_| Self::update_centralities(&batch, round, dht));+ }+ });+ }+}++impl Mapper for CentralityMapper {+ type Job = CentralityJob;++ fn map(&self, _: Self::Job, worker: &CentralityWorker, dht: &DhtConn<CentralityTables>) {+ match self {+ CentralityMapper::SetupCounters => Self::map_setup_counters(worker, dht),+ CentralityMapper::Cardinalities => Self::map_cardinalities(worker, dht),+ CentralityMapper::Centralities => Self::map_centralities(worker, dht),+ }+ }+}diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rsnew file mode 100644index 00000000..1595a8b9--- /dev/null+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/mod.rs@@ -0,0 +1,117 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use crate::hyperloglog::HyperLogLog;+use crate::{ampc::prelude::*, kahan_sum::KahanSum};++use crate::distributed::member::ShardId;+use crate::{ampc::DefaultDhtTable, webgraph};++pub mod coordinator;+mod mapper;+pub mod worker;++pub use coordinator::{CentralityFinish, CentralitySetup};+pub use mapper::CentralityMapper;+pub use worker::{CentralityWorker, RemoteCentralityWorker};++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct Meta {+ round_had_changes: bool,+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct CentralityTables {+ counters: DefaultDhtTable<webgraph::NodeID, HyperLogLog<64>>,+ meta: DefaultDhtTable<(), Meta>,+ centrality: DefaultDhtTable<webgraph::NodeID, KahanSum>,+}++impl CentralityTables {+ pub fn num_shards(&self) -> u64 {+ self.counters.shards().len() as u64+ }+}++impl_dht_tables!(CentralityTables, [counters, meta, centrality]);++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct CentralityJob {+ shard: ShardId,+}++impl Job for CentralityJob {+ type DhtTables = CentralityTables;+ type Worker = CentralityWorker;+ type Mapper = CentralityMapper;++ fn is_schedulable(&self, worker: &RemoteCentralityWorker) -> bool {+ self.shard == worker.shard()+ }+}++#[cfg(test)]+mod tests {+ use tracing_test::traced_test;++ use crate::{free_socket_addr, webgraph::centrality::harmonic::HarmonicCentrality};++ use super::*;++ #[test]+ #[traced_test]+ fn test_simple_graph() {+ let graph = crate::webgraph::tests::test_graph();+ let expected = HarmonicCentrality::calculate(&graph);+ let num_nodes = graph.nodes().count();+ let worker = CentralityWorker::new(1.into(), graph);++ let worker_addr = free_socket_addr();++ std::thread::spawn(move || {+ worker.run(worker_addr).unwrap();+ });++ std::thread::sleep(std::time::Duration::from_secs(1)); // Wait for worker to start++ let remote_worker = RemoteCentralityWorker::new(1.into(), worker_addr);++ assert_eq!(remote_worker.num_nodes(), num_nodes as u64);++ let (dht_shard, dht_addr) = crate::entrypoint::ampc::dht::tests::setup();+ let res = coordinator::build(&[(dht_shard, dht_addr)], vec![remote_worker])+ .run(vec![CentralityJob { shard: 1.into() }], CentralityFinish)+ .unwrap();++ let mut actual = res+ .centrality+ .iter()+ .map(|(n, s)| (n, f64::from(s) / ((num_nodes - 1) as f64)))+ .collect::<Vec<_>>();+ let mut expected = expected.iter().map(|(n, c)| (*n, c)).collect::<Vec<_>>();++ actual.sort_by(|a, b| a.0.cmp(&b.0));+ expected.sort_by(|a, b| a.0.cmp(&b.0));++ for (expected, actual) in expected+ .iter()+ .map(|(_, c)| c)+ .zip(actual.iter().map(|(_, c)| c))+ {+ assert!((expected - actual).abs() < 0.0001);+ }+ }+}diff --git a/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rsnew file mode 100644index 00000000..db27cc5b--- /dev/null+++ b/crates/core/src/entrypoint/ampc/harmonic_centrality/worker.rs@@ -0,0 +1,193 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use crate::{+ ampc::prelude::*,+ bloom::BloomFilter,+ config::HarmonicWorkerConfig,+ distributed::{+ cluster::Cluster,+ member::{Member, Service, ShardId},+ },+ webgraph::{self, Webgraph},+ Result,+};+use std::{+ net::SocketAddr,+ sync::{atomic::AtomicU64, Arc, Mutex},+};++use super::CentralityJob;++pub struct CentralityWorker {+ shard: ShardId,+ graph: Webgraph,+ changed_nodes: Arc<Mutex<BloomFilter>>,+ round: AtomicU64,+}++impl CentralityWorker {+ pub fn new(shard: ShardId, graph: Webgraph) -> Self {+ let num_nodes = graph.estimate_num_nodes() as u64;+ let mut changed_nodes = BloomFilter::new(num_nodes, 0.05);++ for node in graph.nodes() {+ changed_nodes.insert(node.as_u64());+ }++ Self {+ shard,+ graph,+ changed_nodes: Arc::new(Mutex::new(changed_nodes)),+ round: AtomicU64::new(0),+ }+ }++ pub fn shard(&self) -> ShardId {+ self.shard+ }++ pub fn graph(&self) -> &Webgraph {+ &self.graph+ }++ pub fn changed_nodes(&self) -> &Arc<Mutex<BloomFilter>> {+ &self.changed_nodes+ }++ pub fn round(&self) -> u64 {+ self.round.load(std::sync::atomic::Ordering::Relaxed)+ }++ pub fn inc_round(&self) -> u64 {+ self.round+ .fetch_add(1, std::sync::atomic::Ordering::Relaxed)+ }+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct GetShard;++impl Message<CentralityWorker> for GetShard {+ type Response = ShardId;++ fn handle(self, worker: &CentralityWorker) -> Self::Response {+ worker.shard+ }+}+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct NumNodes;++impl Message<CentralityWorker> for NumNodes {+ type Response = u64;++ fn handle(self, worker: &CentralityWorker) -> Self::Response {+ worker.graph.estimate_num_nodes() as u64+ }+}++#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]+pub struct BatchId2Node(Vec<webgraph::NodeID>);++impl Message<CentralityWorker> for BatchId2Node {+ type Response = Vec<(webgraph::NodeID, webgraph::Node)>;++ fn handle(self, worker: &CentralityWorker) -> Self::Response {+ self.0+ .iter()+ .filter_map(|id| worker.graph.id2node(id).map(|node| (*id, node)))+ .collect()+ }+}++impl_worker!(CentralityJob, RemoteCentralityWorker => CentralityWorker, [GetShard, NumNodes, BatchId2Node]);++#[derive(Clone)]+pub struct RemoteCentralityWorker {+ shard: ShardId,+ addr: SocketAddr,+}++impl RemoteCentralityWorker {+ pub fn new(shard: ShardId, addr: SocketAddr) -> Self {+ Self { shard, addr }+ }++ pub fn shard(&self) -> ShardId {+ self.shard+ }++ pub fn num_nodes(&self) -> u64 {+ self.send(NumNodes)+ }++ pub fn batch_id2node(+ &self,+ id: Vec<webgraph::NodeID>,+ ) -> Vec<(webgraph::NodeID, webgraph::Node)> {+ self.send(BatchId2Node(id))+ }+}++impl RemoteWorker for RemoteCentralityWorker {+ type Job = CentralityJob;++ fn remote_addr(&self) -> SocketAddr {+ self.addr+ }+}++fn start_gossip_cluster_thread(config: HarmonicWorkerConfig) {+ std::thread::spawn(move || {+ let rt = tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap();++ rt.block_on(async {+ let _cluster = Cluster::join(+ Member {+ id: config.gossip.cluster_id,+ service: Service::HarmonicWorker {+ host: config.host,+ shard: config.shard,+ },+ },+ config.gossip.addr,+ config.gossip.seed_nodes.unwrap_or_default(),+ )+ .await;++ // need to keep tokio runtime alive+ // otherwise the spawned task in Cluster::join will be dropped+ loop {+ tokio::time::sleep(std::time::Duration::from_secs(1)).await;+ }+ });+ });+}++pub fn run(config: HarmonicWorkerConfig) -> Result<()> {+ let tokio_conf = config.clone();++ let graph = Webgraph::builder(config.graph_path).open();+ let worker = CentralityWorker::new(config.shard, graph);+ start_gossip_cluster_thread(tokio_conf);++ worker.run(config.host)?;++ Ok(())+}diff --git a/crates/core/src/entrypoint/ampc/mod.rs b/crates/core/src/entrypoint/ampc/mod.rsnew file mode 100644index 00000000..dd3bfb4d--- /dev/null+++ b/crates/core/src/entrypoint/ampc/mod.rs@@ -0,0 +1,18 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>.++pub mod dht;+pub mod harmonic_centrality;diff --git a/crates/core/src/entrypoint/centrality.rs b/crates/core/src/entrypoint/centrality.rsindex d3f7bdd0..b845c38a 100644--- a/crates/core/src/entrypoint/centrality.rs+++ b/crates/core/src/entrypoint/centrality.rs@@ -15,37 +15,22 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use anyhow::Result;-use std::{cmp::Reverse, fs::File, path::Path};+use std::{cmp::Reverse, path::Path};use crate::{external_sort::ExternalSorter,kv::{rocksdb_store::RocksDbStore, Kv},ranking::inbound_similarity::InboundSimilarity,webgraph::{- centrality::{approx_harmonic::ApproxHarmonic, harmonic::HarmonicCentrality},- Node, WebgraphBuilder,+ centrality::{+ approx_harmonic::ApproxHarmonic, harmonic::HarmonicCentrality, store_csv,+ store_harmonic, TopNodes,+ },+ WebgraphBuilder,},SortableFloat,};-fn store_csv<P: AsRef<Path>>(data: Vec<(Node, f64)>, output: P) {- let csv_file = File::options()- .write(true)- .create(true)- .truncate(true)- .open(output)- .unwrap();-- let mut data = data;- data.sort_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));- let mut wtr = csv::Writer::from_writer(csv_file);- for (node, centrality) in data {- wtr.write_record(&[node.as_str().to_string(), centrality.to_string()])- .unwrap();- }- wtr.flush().unwrap();-}-pub struct Centrality;impl Centrality {@@ -56,35 +41,16 @@ impl Centrality {);let graph = WebgraphBuilder::new(webgraph_path).single_threaded().open();let harmonic_centrality = HarmonicCentrality::calculate(&graph);- let store = RocksDbStore::open(base_output.as_ref().join("harmonic"));-- for (node_id, centrality) in harmonic_centrality.iter() {- store.insert(*node_id, centrality);- }- store.flush();-- let rank_store: RocksDbStore<crate::webgraph::NodeID, u64> =- RocksDbStore::open(base_output.as_ref().join("harmonic_rank"));- let mut top_harmonics = Vec::new();- for (rank, node, centrality) in ExternalSorter::new()- .with_chunk_size(100_000_000)- .sort(- harmonic_centrality- .iter()- .map(|(node_id, centrality)| (Reverse(SortableFloat(centrality)), *node_id)),- )- .unwrap()- .enumerate()- .map(|(rank, (Reverse(SortableFloat(centrality)), node_id))| {- (rank, node_id, centrality)- })- {- rank_store.insert(node, rank as u64);+ let store = store_harmonic(+ harmonic_centrality.iter().map(|(n, c)| (*n, c)),+ base_output.as_ref(),+ );- if top_harmonics.len() < 1_000_000 {- top_harmonics.push((graph.id2node(&node).unwrap(), centrality));- }- }+ let top_harmonics =+ crate::webgraph::centrality::top_nodes(&store, TopNodes::Top(1_000_000))+ .into_iter()+ .map(|(n, c)| (graph.id2node(&n).unwrap(), c))+ .collect();store_csv(top_harmonics, base_output.as_ref().join("harmonic.csv"));}diff --git a/crates/core/src/entrypoint/crawler.rs b/crates/core/src/entrypoint/crawler.rsindex 2e46552c..ad701bf6 100644--- a/crates/core/src/entrypoint/crawler.rs+++ b/crates/core/src/entrypoint/crawler.rs@@ -16,14 +16,12 @@use std::{net::SocketAddr, sync::Arc};-use serde::{Deserialize, Serialize};-use crate::{config,crawler::{self, planner::CrawlPlanner, CrawlCoordinator, Crawler},+ distributed::sonic::service::sonic_service,distributed::sonic::service::Message,kv::rocksdb_store::RocksDbStore,- sonic_service,webgraph::WebgraphBuilder,Result,};@@ -98,7 +96,9 @@ pub mod router {sonic_service!(RouterService, [NewJob]);- #[derive(Debug, Clone, Serialize, Deserialize)]+ #[derive(+ Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,+ )]pub struct NewJob {}impl Message<RouterService> for NewJob {@@ -121,7 +121,9 @@ pub mod coordinator {sonic_service!(CoordinatorService, [GetJob]);- #[derive(Debug, Clone, Serialize, Deserialize)]+ #[derive(+ Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,+ )]pub struct GetJob {}impl Message<CoordinatorService> for GetJob {diff --git a/crates/core/src/entrypoint/entity.rs b/crates/core/src/entrypoint/entity.rsindex f3c4e9da..58ad9f4c 100644--- a/crates/core/src/entrypoint/entity.rs+++ b/crates/core/src/entrypoint/entity.rs@@ -243,7 +243,6 @@ mod tests {use super::*;- #[macro_export]macro_rules! expect {($($t:tt)*) => {|res| assert_snapshot!(res, $($t)*)diff --git a/crates/core/src/entrypoint/entity_search_server.rs b/crates/core/src/entrypoint/entity_search_server.rsindex 397d8bad..fabb815a 100644--- a/crates/core/src/entrypoint/entity_search_server.rs+++ b/crates/core/src/entrypoint/entity_search_server.rs@@ -14,10 +14,9 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use serde::{Deserialize, Serialize};-use crate::{config,+ distributed::sonic::service::sonic_service,distributed::{cluster::Cluster,member::{Member, Service},@@ -25,7 +24,6 @@ use crate::{},entity_index::EntityIndex,image_store::Image,- sonic_service,};use anyhow::Result;@@ -58,7 +56,7 @@ impl SearchService {}}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct Search {pub query: String,}@@ -70,7 +68,7 @@ impl sonic::service::Message<SearchService> for Search {}}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct GetEntityImage {pub image_id: String,pub max_width: Option<u64>,diff --git a/crates/core/src/entrypoint/feed_indexer.rs b/crates/core/src/entrypoint/feed_indexer.rsindex 50254027..aecc9629 100644--- a/crates/core/src/entrypoint/feed_indexer.rs+++ b/crates/core/src/entrypoint/feed_indexer.rs@@ -22,7 +22,7 @@ use std::{use anyhow::Result;use itertools::Itertools;use rayon::prelude::*;-use serde::{Deserialize, Serialize};+use tracing::info;use url::Url;@@ -30,18 +30,17 @@ use crate::{config::{FeedIndexingConfig, WarcSource},entrypoint::download_all_warc_files,feed::{index::FeedIndex, Feed, FeedKind},- mapreduce::{Map, Worker},warc::PayloadType,};-#[derive(Debug, Serialize, Deserialize)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct Job {pub source_config: WarcSource,pub warc_path: String,pub base_path: String,}-#[derive(Debug, Serialize, Deserialize)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct IndexPointer(PathBuf);impl From<String> for IndexPointer {@@ -50,15 +49,6 @@ impl From<String> for IndexPointer {}}-impl Worker for IndexingWorker {}--impl Map<IndexingWorker, IndexPointer> for Job {- fn map(&self, worker: &IndexingWorker) -> IndexPointer {- let index = worker.process_job(self);- IndexPointer(index.path)- }-}-pub struct IndexingWorker {}impl IndexingWorker {fn process_job(&self, job: &Job) -> FeedIndex {@@ -131,10 +121,7 @@ pub fn build(config: FeedIndexingConfig) -> Result<()> {warc_path,base_path: config.output_path.clone(),})- .map(|job| {- let pointer: IndexPointer = job.map(&worker);- pointer- })+ .map(|job| IndexPointer(worker.process_job(&job).path)).collect();merge(indexes)?;diff --git a/crates/core/src/entrypoint/indexer/job.rs b/crates/core/src/entrypoint/indexer/job.rsindex a75b5dc0..9e2d7de7 100644--- a/crates/core/src/entrypoint/indexer/job.rs+++ b/crates/core/src/entrypoint/indexer/job.rs@@ -17,7 +17,7 @@use std::path::Path;use itertools::Itertools;-use serde::{Deserialize, Serialize};+use tokio::pin;use tracing::{info, trace, warn};@@ -28,7 +28,7 @@ use crate::warc::PayloadType;use super::{IndexableWebpage, IndexingWorker};-#[derive(Debug, Serialize, Deserialize)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct Job {pub source_config: config::WarcSource,pub warc_path: String,@@ -36,7 +36,9 @@ pub struct Job {pub settings: JobSettings,}-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]+#[derive(+ Debug, Clone, Copy, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,+)]pub struct JobSettings {pub host_centrality_threshold: Option<f64>,pub minimum_clean_words: Option<usize>,diff --git a/crates/core/src/entrypoint/indexer/mod.rs b/crates/core/src/entrypoint/indexer/mod.rsindex 9693cea0..b6491cf9 100644--- a/crates/core/src/entrypoint/indexer/mod.rs+++ b/crates/core/src/entrypoint/indexer/mod.rs@@ -22,7 +22,6 @@ use rayon::prelude::*;use std::thread;use itertools::Itertools;-use serde::{Deserialize, Serialize};pub use crate::entrypoint::indexer::indexable_webpage::IndexableWebpage;pub use crate::entrypoint::indexer::job::{Job, JobSettings};@@ -30,10 +29,9 @@ pub use crate::entrypoint::indexer::worker::IndexingWorker;use crate::config::{self, WarcSource};use crate::index::Index;-use crate::mapreduce::{Map, Reduce, Worker};use crate::Result;-#[derive(Debug, Serialize, Deserialize)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct IndexPointer(String);impl From<String> for IndexPointer {@@ -42,28 +40,6 @@ impl From<String> for IndexPointer {}}-impl Worker for IndexingWorker {}--impl Map<IndexingWorker, IndexPointer> for Job {- fn map(&self, worker: &IndexingWorker) -> IndexPointer {- let index = self.process(worker);- IndexPointer(index.path)- }-}--impl Reduce<Index> for Index {- fn reduce(self, element: Index) -> Self {- let other = element;- let other_path = other.path.clone();-- let res = self.merge(other);-- std::fs::remove_dir_all(other_path).unwrap();-- res- }-}-pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {let warc_paths = config.warc_source.paths()?;@@ -85,10 +61,7 @@ pub fn run(config: &config::IndexingLocalConfig) -> Result<()> {batch_size: config.batch_size,},})- .map(|job| {- let pointer: IndexPointer = job.map(&worker);- pointer- })+ .map(|job| IndexPointer(job.process(&worker).path)).collect();merge(indexes)?;diff --git a/crates/core/src/entrypoint/live_index.rs b/crates/core/src/entrypoint/live_index.rsindex c3a567a4..44dd1236 100644--- a/crates/core/src/entrypoint/live_index.rs+++ b/crates/core/src/entrypoint/live_index.rs@@ -18,6 +18,7 @@ use std::{path::Path, sync::Arc};use crate::{config::{LiveIndexConfig, LiveIndexSchedulerConfig},+ distributed::sonic::service::sonic_service,distributed::{cluster::Cluster,member::{Member, Service},@@ -29,7 +30,6 @@ use crate::{live_index::{Index, IndexManager},ranking::inbound_similarity::InboundSimilarity,searcher::{InitialWebsiteResult, LocalSearcher},- sonic_service,webgraph::WebgraphBuilder,};use anyhow::Result;diff --git a/crates/core/src/entrypoint/mod.rs b/crates/core/src/entrypoint/mod.rsindex 8f63e4b6..d98533c7 100644--- a/crates/core/src/entrypoint/mod.rs+++ b/crates/core/src/entrypoint/mod.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -15,6 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.//! The entrypoint module contains all entrypoints that runs the executables.+pub mod ampc;pub mod api;pub mod autosuggest_scrape;mod centrality;diff --git a/crates/core/src/entrypoint/search_server.rs b/crates/core/src/entrypoint/search_server.rsindex 0aba25db..94223c86 100644--- a/crates/core/src/entrypoint/search_server.rs+++ b/crates/core/src/entrypoint/search_server.rs@@ -16,12 +16,12 @@use std::{collections::HashMap, path::Path};-use serde::{Deserialize, Serialize};use tracing::info;use url::Url;use crate::{config,+ distributed::sonic::service::sonic_service,distributed::{cluster::Cluster,member::{Member, Service},@@ -35,7 +35,7 @@ use crate::{models::{lambdamart::LambdaMART, linear::LinearRegression},},searcher::{InitialWebsiteResult, LocalSearcher, SearchQuery},- sonic_service, Result,+ Result,};sonic_service!(@@ -103,7 +103,7 @@ impl SearchService {}}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct RetrieveWebsites {pub websites: Vec<inverted_index::WebpagePointer>,pub query: String,@@ -118,7 +118,7 @@ impl sonic::service::Message<SearchService> for RetrieveWebsites {}}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct Search {pub query: SearchQuery,}@@ -129,7 +129,7 @@ impl sonic::service::Message<SearchService> for Search {}}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct GetWebpage {pub url: String,}@@ -140,12 +140,13 @@ impl sonic::service::Message<SearchService> for GetWebpage {}}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct GetHomepageDescriptions {+ #[bincode(with_serde)]pub urls: Vec<Url>,}impl sonic::service::Message<SearchService> for GetHomepageDescriptions {- type Response = HashMap<Url, String>;+ type Response = crate::bincode_utils::SerdeCompat<HashMap<Url, String>>;async fn handle(self, server: &SearchService) -> Self::Response {let mut result = HashMap::with_capacity(self.urls.len());@@ -157,7 +158,7 @@ impl sonic::service::Message<SearchService> for GetHomepageDescriptions {}}- result+ crate::bincode_utils::SerdeCompat(result)}}diff --git a/crates/core/src/entrypoint/web_spell.rs b/crates/core/src/entrypoint/web_spell.rsindex 33b187fc..2e31fe23 100644--- a/crates/core/src/entrypoint/web_spell.rs+++ b/crates/core/src/entrypoint/web_spell.rs@@ -19,14 +19,13 @@ use std::path::{Path, PathBuf};use anyhow::Result;use fnv::FnvHashMap;use itertools::Itertools;-use serde::{Deserialize, Serialize};+use tracing::{debug, info};use whatlang::Lang;use crate::{config::{self, WebSpellConfig},entrypoint::download_all_warc_files,- mapreduce::Worker,web_spell::{FirstTrainer, FirstTrainerResult, SecondTrainer},webpage::Html,};@@ -118,14 +117,12 @@ impl SpellWorker {}}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct Job {pub source_config: config::WarcSource,pub warc_path: String,}-impl Worker for SpellWorker {}-pub fn run(config: WebSpellConfig) -> Result<()> {let warc_paths = config.warc_source.paths()?;diff --git a/crates/core/src/entrypoint/webgraph.rs b/crates/core/src/entrypoint/webgraph.rsindex 6d5ad890..a6744e0f 100644--- a/crates/core/src/entrypoint/webgraph.rs+++ b/crates/core/src/entrypoint/webgraph.rs@@ -17,23 +17,22 @@ use crate::{config::WarcSource,config::{self, WebgraphConstructConfig},entrypoint::download_all_warc_files,- mapreduce::Worker,webgraph::{self, Node, WebgraphWriter},webpage::{url_ext::UrlExt, Html},Result,};use itertools::Itertools;-use serde::{Deserialize, Serialize};+use std::{fs, path::Path};use tokio::pin;use tracing::{info, trace};-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]struct GraphPointer {path: String,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]pub enum JobConfig {Http(config::HttpConfig),Local(config::LocalConfig),@@ -60,7 +59,7 @@ impl From<JobConfig> for config::WarcSource {}}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]pub struct Job {pub config: JobConfig,pub warc_paths: Vec<String>,@@ -148,8 +147,6 @@ impl WebgraphWorker {}}-impl Worker for WebgraphWorker {}-pub struct Webgraph {}impl Webgraph {diff --git a/crates/core/src/entrypoint/webgraph_server.rs b/crates/core/src/entrypoint/webgraph_server.rsindex db87f7c0..04edd45e 100644--- a/crates/core/src/entrypoint/webgraph_server.rs+++ b/crates/core/src/entrypoint/webgraph_server.rs@@ -18,8 +18,6 @@ use std::net::SocketAddr;use std::sync::Arc;use itertools::Itertools;-use serde::Deserialize;-use serde::Serialize;use tracing::info;use url::Url;use utoipa::ToSchema;@@ -29,12 +27,12 @@ use crate::config::WebgraphGranularity;use crate::distributed::cluster::Cluster;use crate::distributed::member::Member;use crate::distributed::member::Service;+use crate::distributed::sonic::service::sonic_service;use crate::distributed::sonic::service::Message;use crate::ranking::inbound_similarity::InboundSimilarity;use crate::searcher::DistributedSearcher;use crate::searcher::SearchClient;use crate::similar_hosts::SimilarHostsFinder;-use crate::sonic_service;use crate::webgraph::Compression;use crate::webgraph::FullEdge;use crate::webgraph::Node;@@ -42,7 +40,7 @@ use crate::webgraph::Webgraph;use crate::webgraph::WebgraphBuilder;use crate::Result;-#[derive(serde::Serialize, serde::Deserialize, ToSchema)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema)]#[serde(rename_all = "camelCase")]pub struct ScoredHost {pub host: String,@@ -64,13 +62,13 @@ sonic_service!([SimilarHosts, Knows, IngoingLinks, OutgoingLinks]);-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct SimilarHosts {pub hosts: Vec<String>,pub top_n: usize,}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct Knows {pub host: String,}@@ -145,7 +143,7 @@ impl Message<WebGraphService> for Knows {}}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct IngoingLinks {pub node: Node,}@@ -164,7 +162,7 @@ impl Message<WebGraphService> for IngoingLinks {}}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct OutgoingLinks {pub node: Node,}diff --git a/crates/core/src/enum_map.rs b/crates/core/src/enum_map.rsindex 45eeb56f..6ac1e345 100644--- a/crates/core/src/enum_map.rs+++ b/crates/core/src/enum_map.rs@@ -14,8 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use serde::{Deserialize, Serialize};-pub trait InsertEnumMapKey: Sized {fn into_usize(self) -> usize;}@@ -24,7 +22,7 @@ pub trait GetEnumMapKey: Sized {fn from_usize(value: usize) -> Option<Self>;}-#[derive(Clone, Debug, Serialize, Deserialize)]+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct EnumMap<K: InsertEnumMapKey, V> {inner: Vec<Option<V>>,len: usize,@@ -125,7 +123,7 @@ where}}-#[derive(Clone, Debug, Serialize, Deserialize)]+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct EnumSet<K: InsertEnumMapKey> {map: EnumMap<K, ()>,}diff --git a/crates/core/src/external_sort.rs b/crates/core/src/external_sort.rsindex 31f7bc67..e1164b44 100644--- a/crates/core/src/external_sort.rs+++ b/crates/core/src/external_sort.rs@@ -52,12 +52,12 @@ impl<T> Chunk<T> {fn write_into(self, file: &mut File) -> Result<()>where- T: serde::Serialize,+ T: bincode::Encode,{let mut wrt = BufWriter::new(file);for item in self.data {- let bytes = bincode::serialize(&item)?;+ let bytes = bincode::encode_to_vec(&item, bincode::config::standard())?;let size = bytes.len() as u64;wrt.write_all(&size.to_le_bytes())?;@@ -71,7 +71,7 @@ impl<T> Chunk<T> {fn store(mut self, mut file: TempFile) -> Result<StoredChunk<T>>where- T: serde::Serialize + Ord,+ T: bincode::Encode + Ord,{self.sort();self.write_into(&mut file.inner)?;@@ -161,7 +161,7 @@ impl<T> StoredChunk<T> {fn next(&mut self) -> Option<T>where- T: serde::de::DeserializeOwned,+ T: bincode::Decode,{self.buf.clear();self.buf.resize(u64::BITS as usize / 8, 0);@@ -175,7 +175,7 @@ impl<T> StoredChunk<T> {self.data.read_exact(&mut self.buf).ok()?;- let next = bincode::deserialize(&self.buf).ok()?;+ let (next, _) = bincode::decode_from_slice(&self.buf, bincode::config::standard()).ok()?;Some(next)}@@ -208,7 +208,7 @@ impl<T> ExternalSorter<T> {pub fn sort<I>(self, iter: I) -> Result<SortedIterator<T>>whereI: Iterator<Item = T>,- T: serde::Serialize + serde::de::DeserializeOwned + Ord,+ T: bincode::Encode + bincode::Decode + Ord,{let dir = TempDir::new()?;let mut stored_chunks = Vec::new();@@ -251,7 +251,7 @@ struct Head<T> {impl<T> Head<T>where- T: serde::de::DeserializeOwned,+ T: bincode::Decode,{fn new(mut chunk: StoredChunk<T>) -> Option<Self> {let item = chunk.next()?;@@ -306,7 +306,7 @@ pub struct SortedIterator<T> {impl<T> Iterator for SortedIterator<T>where- T: serde::de::DeserializeOwned + Ord,+ T: bincode::Decode + Ord,{type Item = T;diff --git a/crates/core/src/feed/mod.rs b/crates/core/src/feed/mod.rsindex 1f9ebf9b..8f9ffd6d 100644--- a/crates/core/src/feed/mod.rs+++ b/crates/core/src/feed/mod.rs@@ -22,14 +22,36 @@ pub mod scheduler;pub use parser::parse;-#[derive(Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]+#[derive(+ Debug,+ Clone,+ Copy,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Hash,+)]pub enum FeedKind {Atom,Rss,}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]+#[derive(+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Hash,+)]pub struct Feed {+ #[bincode(with_serde)]pub url: Url,pub kind: FeedKind,}diff --git a/crates/core/src/feed/scheduler.rs b/crates/core/src/feed/scheduler.rsindex d3598bae..9b5a237e 100644--- a/crates/core/src/feed/scheduler.rs+++ b/crates/core/src/feed/scheduler.rs@@ -23,7 +23,7 @@ use url::Url;use crate::{kv::rocksdb_store::RocksDbStore,webgraph::{- centrality::{top_hosts, TopHosts},+ centrality::{top_nodes, TopNodes},NodeID, Webgraph,},webpage::url_ext::UrlExt,@@ -31,10 +31,20 @@ use crate::{use super::{index::FeedIndex, Feed};-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]+#[derive(+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Hash,+)]pub struct Domain(String);-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct DomainFeeds {pub domain: Domain,pub feeds: Vec<Feed>,@@ -46,8 +56,18 @@ impl From<&Url> for Domain {}}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash)]-pub struct SplitId(uuid::Uuid);+#[derive(+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Hash,+)]+pub struct SplitId(#[bincode(with_serde)] uuid::Uuid);impl SplitId {pub fn id(&self) -> uuid::Uuid {@@ -55,7 +75,7 @@ impl SplitId {}}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct Split {pub id: SplitId,pub feeds: Vec<DomainFeeds>,@@ -109,11 +129,11 @@ pub fn schedule(host_graph: &Webgraph,num_splits: u64,) -> Schedule {- let top_hosts = top_hosts(host_centrality, TopHosts::Top(1_000_000));+ let top_hosts = top_nodes(host_centrality, TopNodes::Top(1_000_000));let mut all_feeds = HashMap::new();- for host in top_hosts {+ for (host, _) in top_hosts {let host = host_graph.id2node(&host).unwrap();let url = Url::parse(&format!("http://{}", host.as_str()));diff --git a/crates/core/src/highlighted.rs b/crates/core/src/highlighted.rsindex 2191b4a4..2848eef4 100644--- a/crates/core/src/highlighted.rs+++ b/crates/core/src/highlighted.rs@@ -16,14 +16,32 @@use utoipa::ToSchema;-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, ToSchema)]+#[derive(+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ ToSchema,+)]#[serde(rename_all = "camelCase")]pub enum HighlightedKind {Normal,Highlighted,}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, ToSchema)]+#[derive(+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct HighlightedFragment {pub kind: HighlightedKind,diff --git a/crates/core/src/human_website_annotations.rs b/crates/core/src/human_website_annotations.rsindex 6d326421..416af3ab 100644--- a/crates/core/src/human_website_annotations.rs+++ b/crates/core/src/human_website_annotations.rs@@ -15,22 +15,52 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use itertools::Itertools;-use serde::{Deserialize, Serialize};+use tantivy::schema::Facet;use crate::Result;use std::{collections::{HashMap, HashSet},fs::File,- io::{BufReader, Read, Write},+ io::{BufReader, Write},path::Path,};-#[derive(Debug, Hash, PartialEq, Eq, Clone, Serialize, Deserialize)]+#[derive(Debug, Hash, PartialEq, Eq, Clone, serde::Serialize, serde::Deserialize)]pub struct Topic<const N: usize = 2> {detailed_topics: Vec<String>,}+/// bincode seems to not like default const generics, therefore manual impl+impl<const N: usize> bincode::Encode for Topic<N> {+ fn encode<E: bincode::enc::Encoder>(+ &self,+ encoder: &mut E,+ ) -> Result<(), bincode::error::EncodeError> {+ self.detailed_topics.encode(encoder)+ }+}++impl<const N: usize> bincode::Decode for Topic<N> {+ fn decode<D: bincode::de::Decoder>(+ decoder: &mut D,+ ) -> Result<Self, bincode::error::DecodeError> {+ Ok(Self {+ detailed_topics: Vec::decode(decoder)?,+ })+ }+}++impl<'de, const N: usize> bincode::BorrowDecode<'de> for Topic<N> {+ fn borrow_decode<D: bincode::de::BorrowDecoder<'de>>(+ decoder: &mut D,+ ) -> Result<Self, bincode::error::DecodeError> {+ Ok(Self {+ detailed_topics: Vec::borrow_decode(decoder)?,+ })+ }+}+impl<const N: usize> From<Facet> for Topic<N> {fn from(facet: Facet) -> Self {Self {@@ -57,13 +87,13 @@ impl<const N: usize> Topic<N> {}}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct Info {pub description: String,pub topic: Topic,}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct Mapper(HashMap<String, Info>);impl From<HashMap<String, Info>> for Mapper {@@ -80,7 +110,7 @@ impl Mapper {.write(true).open(path)?;- let bytes = bincode::serialize(&self)?;+ let bytes = bincode::encode_to_vec(self, bincode::config::standard())?;file.write_all(&bytes)?;Ok(())@@ -89,10 +119,10 @@ impl Mapper {pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {let mut reader = BufReader::new(File::open(path)?);- let mut bytes = Vec::new();- reader.read_to_end(&mut bytes)?;-- Ok(bincode::deserialize(&bytes)?)+ Ok(bincode::decode_from_std_read(+ &mut reader,+ bincode::config::standard(),+ )?)}pub fn get(&self, host: &String) -> Option<&Info> {diff --git a/crates/core/src/hyperloglog.rs b/crates/core/src/hyperloglog.rsindex 4fdfd668..e980d581 100644--- a/crates/core/src/hyperloglog.rs+++ b/crates/core/src/hyperloglog.rs@@ -24,8 +24,6 @@ use std::{marker::PhantomData,};-use serde::{Deserialize, Serialize};-pub(crate) const THRESHOLD_DATA_OFFSET: usize = 4;pub(crate) const THRESHOLD_DATA_VEC: &[usize] = &[10, // b = 4@@ -4305,7 +4303,7 @@ pub trait HyperLogLogHasher: std::fmt::Debug + Clone + Copy + Send + Sync + 'stafn hash(item: u64) -> u64;}-#[derive(Clone, Copy, Debug)]+#[derive(Clone, Copy, Debug, bincode::Encode, bincode::Decode)]pub struct FastHasher;impl HyperLogLogHasher for FastHasher {@@ -4327,7 +4325,21 @@ impl HyperLogLogHasher for StableHasher {}}-#[derive(Debug, Clone, Serialize, Deserialize)]+use bytecheck::CheckBytes;+use rkyv::bytecheck;++#[derive(+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ rkyv::Archive,+ rkyv::Serialize,+ rkyv::Deserialize,+)]+#[archive_attr(derive(CheckBytes))]pub struct HyperLogLog<const N: usize, H = FastHasher>whereH: HyperLogLogHasher,diff --git a/crates/core/src/image_store.rs b/crates/core/src/image_store.rsindex be0c789c..fdb4c2c7 100644--- a/crates/core/src/image_store.rs+++ b/crates/core/src/image_store.rs@@ -16,60 +16,54 @@use crate::kv::{rocksdb_store::RocksDbStore, Kv};use crate::Result;+use bincode::de::read::Reader;+use bincode::enc::write::Writer;use image::imageops::FilterType;use image::{DynamicImage, ImageOutputFormat};-use serde::{de, ser::SerializeStruct, Serialize};+use serde::Serialize;use std::io::{Cursor, Read, Seek};use std::path::Path;#[derive(PartialEq, Debug, Clone)]pub struct Image(DynamicImage);-impl Serialize for Image {- fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>- where- S: serde::Serializer,- {+impl bincode::Encode for Image {+ fn encode<E: bincode::enc::Encoder>(+ &self,+ encoder: &mut E,+ ) -> Result<(), bincode::error::EncodeError> {let bytes = self.as_raw_bytes();+ let len = bytes.len() as u64;+ len.encode(encoder)?;+ encoder.writer().write(bytes.as_slice())+ }+}++impl bincode::Decode for Image {+ fn decode<D: bincode::de::Decoder>(+ decoder: &mut D,+ ) -> Result<Self, bincode::error::DecodeError> {+ let len = u64::decode(decoder)?;+ let mut raw = vec![0; len as usize];+ decoder.reader().read(&mut raw)?;- let mut image = serializer.serialize_struct("image", 1)?;- image.serialize_field("0", &bytes)?;- image.end()+ Ok(Image(+ image::load_from_memory(&raw).expect("bytes does not seem to represent an image"),+ ))}}-impl<'de> de::Deserialize<'de> for Image {- fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>- where- D: serde::Deserializer<'de>,- {- struct ImageVisitor;-- impl<'de> de::Visitor<'de> for ImageVisitor {- type Value = Image;-- fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {- formatter.write_str("a serialized `Image` struct")- }-- fn visit_seq<A>(self, mut seq: A) -> std::result::Result<Self::Value, A::Error>- where- A: serde::de::SeqAccess<'de>,- {- let raw = seq- .next_element()?- .ok_or_else(|| de::Error::invalid_length(0, &self))?;-- let image = Image(- image::load_from_memory(raw)- .expect("bytes does not seem to represent an image"),- );-- Ok(image)- }- }+impl<'de> bincode::BorrowDecode<'de> for Image {+ fn borrow_decode<D: bincode::de::BorrowDecoder<'de>>(+ decoder: &mut D,+ ) -> Result<Self, bincode::error::DecodeError> {+ let len = u64::borrow_decode(decoder)?;+ let mut raw = vec![0; len as usize];+ decoder.reader().read(&mut raw)?;- deserializer.deserialize_struct("image", &["0"], ImageVisitor)+ Ok(Image(+ image::load_from_memory(&raw).expect("bytes does not seem to represent an image"),+ ))}}@@ -244,8 +238,9 @@ mod tests {ImageBuffer::from_pixel(2, 2, image::Rgb::<u16>([u16::MAX, u16::MAX, u16::MAX])).into(),);- let bytes = bincode::serialize(&image).unwrap();- let decoded_image = bincode::deserialize(&bytes).unwrap();+ let bytes = bincode::encode_to_vec(&image, bincode::config::standard()).unwrap();+ let (decoded_image, _) =+ bincode::decode_from_slice(&bytes, bincode::config::standard()).unwrap();assert_eq!(image, decoded_image);}diff --git a/crates/core/src/intmap.rs b/crates/core/src/intmap.rsindex a4d737f8..a295497d 100644--- a/crates/core/src/intmap.rs+++ b/crates/core/src/intmap.rs@@ -14,8 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use serde::{Deserialize, Serialize};-pub trait Key: PartialOrd + Ord + PartialEq + Copy + std::fmt::Debug {const BIG_PRIME: Self;@@ -35,7 +33,7 @@ impl Key for u64 {}}-#[derive(Serialize, Deserialize, Debug)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug)]pub struct IntMap<K: Key, V> {bins: Vec<Vec<(K, V)>>,len: usize,diff --git a/crates/core/src/inverted_index/mod.rs b/crates/core/src/inverted_index/mod.rsindex 94de2bb2..7343b43e 100644--- a/crates/core/src/inverted_index/mod.rs+++ b/crates/core/src/inverted_index/mod.rs@@ -32,7 +32,6 @@ mod search;pub use indexing::merge_tantivy_segments;use chrono::NaiveDateTime;-use serde::{Deserialize, Serialize};use tantivy::directory::MmapDirectory;@@ -61,20 +60,31 @@ use std::fs;use std::path::Path;use std::sync::Arc;-#[derive(Debug, Serialize, Deserialize)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct InitialSearchResult {pub num_websites: Option<usize>,pub top_websites: Vec<WebpagePointer>,}-#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, PartialEq,+)]pub struct WebpagePointer {pub score: Score,pub hashes: Hashes,pub address: DocAddress,}-#[derive(Debug, Serialize, Deserialize, Clone, Copy, PartialEq)]+#[derive(+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Clone,+ Copy,+ PartialEq,+)]pub struct DocAddress {pub segment: u32,pub doc_id: u32,@@ -202,13 +212,22 @@ impl InvertedIndex {}}-#[derive(Debug, Serialize)]+#[derive(Debug, serde::Serialize, bincode::Encode)]pub struct SearchResult {pub num_docs: Option<usize>,pub documents: Vec<RetrievedWebpage>,}-#[derive(Default, Debug, Clone, Serialize, Deserialize, PartialEq)]+#[derive(+ Default,+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+)]pub struct RetrievedWebpage {pub title: String,pub url: String,@@ -217,6 +236,7 @@ pub struct RetrievedWebpage {pub dirty_body: String,pub description: Option<String>,pub dmoz_description: Option<String>,+ #[bincode(with_serde)]pub updated_time: Option<NaiveDateTime>,pub schema_org: Vec<schema_org::Item>,pub region: Region,diff --git a/crates/core/src/kahan_sum.rs b/crates/core/src/kahan_sum.rsindex 4ad3fb49..7c73d272 100644--- a/crates/core/src/kahan_sum.rs+++ b/crates/core/src/kahan_sum.rs@@ -16,7 +16,16 @@use std::ops::{Add, AddAssign};-#[derive(Default, Clone, Copy, Debug)]+#[derive(+ Default,+ Clone,+ Copy,+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct KahanSum {sum: f64,err: f64,diff --git a/crates/core/src/kv/mod.rs b/crates/core/src/kv/mod.rsindex 84470c8f..b55a34d2 100644--- a/crates/core/src/kv/mod.rs+++ b/crates/core/src/kv/mod.rs@@ -14,31 +14,36 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use serde::{de::DeserializeOwned, Serialize};-pub mod rocksdb_store;pub trait Kv<K, V>where- K: Serialize + DeserializeOwned,- V: Serialize + DeserializeOwned,+ K: bincode::Encode + bincode::Decode,+ V: bincode::Encode + bincode::Decode,Self: Send + Sync,{+ fn approx_len(&self) -> usize;fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>>;fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>);fn flush(&self);fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (K, V)> + 'a>;fn get(&self, key: &K) -> Option<V> {- let key_bytes = bincode::serialize(key).expect("failed to serialize key");+ let key_bytes = bincode::encode_to_vec(key, bincode::config::standard())+ .expect("failed to serialize key");- self.get_raw(&key_bytes)- .map(|bytes| bincode::deserialize(&bytes).expect("failed to deserialize stored value"))+ self.get_raw(&key_bytes).map(|bytes| {+ let (res, _) = bincode::decode_from_slice(&bytes, bincode::config::standard())+ .expect("failed to deserialize stored value");+ res+ })}fn insert(&self, key: K, value: V) {- let key_bytes = bincode::serialize(&key).expect("failed to serialize key");- let val_bytes = bincode::serialize(&value).expect("failed to serialize value");+ let key_bytes = bincode::encode_to_vec(&key, bincode::config::standard())+ .expect("failed to serialize key");+ let val_bytes = bincode::encode_to_vec(&value, bincode::config::standard())+ .expect("failed to serialize value");self.insert_raw(key_bytes, val_bytes);}diff --git a/crates/core/src/kv/rocksdb_store.rs b/crates/core/src/kv/rocksdb_store.rsindex e0d7cc07..f36d3089 100644--- a/crates/core/src/kv/rocksdb_store.rs+++ b/crates/core/src/kv/rocksdb_store.rs@@ -20,14 +20,13 @@ use rocksdb::{BlockBasedOptions, DBIteratorWithThreadMode, DBWithThreadMode, IteratorMode, Options,SingleThreaded, DB,};-use serde::{de::DeserializeOwned, Serialize};use crate::kv::Kv;pub struct RocksDbStore<K, V>where- K: Serialize + DeserializeOwned,- V: Serialize + DeserializeOwned,+ K: bincode::Encode + bincode::Decode,+ V: bincode::Encode + bincode::Decode,{db: DB,_cache: rocksdb::Cache,@@ -36,8 +35,8 @@ whereimpl<K, V> RocksDbStore<K, V>where- K: Serialize + DeserializeOwned,- V: Serialize + DeserializeOwned,+ K: bincode::Encode + bincode::Decode,+ V: bincode::Encode + bincode::Decode,{fn options(cache: &rocksdb::Cache) -> Options {let mut options = Options::default();@@ -113,8 +112,8 @@ whereimpl<K, V> Kv<K, V> for RocksDbStore<K, V>where- K: Serialize + DeserializeOwned + 'static + Send + Sync,- V: Serialize + DeserializeOwned + 'static + Send + Sync,+ K: bincode::Encode + bincode::Decode + 'static + Send + Sync,+ V: bincode::Encode + bincode::Decode + 'static + Send + Sync,{fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {let mut opts = rocksdb::ReadOptions::default();@@ -154,12 +153,21 @@ wherevalue: PhantomData,})}++ fn approx_len(&self) -> usize {+ self.db+ .property_value("rocksdb.estimate-num-keys")+ .unwrap()+ .unwrap()+ .parse()+ .unwrap()+ }}pub struct IntoIter<'a, K, V>where- K: Serialize + DeserializeOwned,- V: Serialize + DeserializeOwned,+ K: bincode::Encode + bincode::Decode,+ V: bincode::Encode + bincode::Decode,{inner: DBIteratorWithThreadMode<'a, DBWithThreadMode<SingleThreaded>>,key: PhantomData<K>,@@ -168,8 +176,8 @@ whereimpl<'a, K, V> Iterator for IntoIter<'a, K, V>where- K: Serialize + DeserializeOwned,- V: Serialize + DeserializeOwned,+ K: bincode::Encode + bincode::Decode,+ V: bincode::Encode + bincode::Decode,{type Item = (K, V);@@ -178,10 +186,14 @@ where.next().and_then(|r| r.ok()).map(|(key_bytes, value_bytes)| {- (- bincode::deserialize(&key_bytes).expect("Failed to deserialize key"),- bincode::deserialize(&value_bytes).expect("Failed to deserialize value"),- )+ let (key, _) = bincode::decode_from_slice(&key_bytes, bincode::config::standard())+ .expect("Failed to deserialize key");++ let (val, _) =+ bincode::decode_from_slice(&value_bytes, bincode::config::standard())+ .expect("Failed to deserialize val");++ (key, val)})}}diff --git a/crates/core/src/lib.rs b/crates/core/src/lib.rsindex ce0d9f1e..dac7c184 100644--- a/crates/core/src/lib.rs+++ b/crates/core/src/lib.rs@@ -1,5 +1,5 @@// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS+// Copyright (C) 2024 Stract ApS//// This program is free software: you can redistribute it and/or modify// it under the terms of the GNU Affero General Public License as@@ -28,18 +28,18 @@// #![allow(clippy::module_name_repetitions)] // maybe we should remove this later// #![allow(clippy::missing_errors_doc)]-use serde::{Deserialize, Serialize};use std::path::PathBuf;use thiserror::Error;pub mod entrypoint;pub mod inverted_index;-pub mod mapreduce;+pub mod ampc;mod api;pub mod autosuggest;pub mod bangs;+mod bincode_utils;mod bloom;mod collector;pub mod config;@@ -208,7 +208,9 @@ pub fn combine_u64s(nums: [u64; 2]) -> u128 {((nums[0] as u128) << 64) | (nums[1] as u128)}-#[derive(Debug, Clone, Copy, Serialize, Deserialize)]+#[derive(+ Debug, Clone, Copy, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,+)]pub struct SortableFloat(f64);impl From<f64> for SortableFloat {@@ -255,7 +257,6 @@ mod tests {}}-#[macro_export]macro_rules! enum_dispatch_from_discriminant {($discenum:ident => $enum:ident, [$($disc:ident),*$(,)?]) => {impl From<$discenum> for $enum {@@ -269,3 +270,10 @@ macro_rules! enum_dispatch_from_discriminant {}};}++pub(crate) use enum_dispatch_from_discriminant;++const XXH3_SECRET: &[u8] = &xxhash_rust::const_xxh3::const_custom_default_secret(42);+pub fn fast_stable_hash(t: &[u8]) -> u64 {+ xxhash_rust::xxh3::xxh3_64_with_secret(t, XXH3_SECRET)+}diff --git a/crates/core/src/main.rs b/crates/core/src/main.rsindex d531c5d7..362a3d8c 100644--- a/crates/core/src/main.rs+++ b/crates/core/src/main.rs@@ -125,6 +125,25 @@ enum Commands {WebSpell {config_path: String,},++ // Commands to compute distributed graph algorithms.+ Ampc {+ #[clap(subcommand)]+ options: AmpcOptions,+ },+}++#[derive(Subcommand)]+enum AmpcOptions {+ /// Start a node for the distributed hash table (DHT).+ Dht { config_path: String },++ /// Start a worker to compute the harmonic centrality of a graph.+ HarmonicWorker { config_path: String },++ /// Start a coordinator to distribute the harmonic centrality computation.+ /// Workers needs to be started before the coordinator.+ HarmonicCoordinator { config_path: String },}#[derive(Subcommand)]@@ -400,6 +419,24 @@ fn main() -> Result<()> {let config: config::WebSpellConfig = load_toml_config(config_path);entrypoint::web_spell::run(config)?;}+ Commands::Ampc { options } => match options {+ AmpcOptions::Dht { config_path } => {+ let config: config::DhtConfig = load_toml_config(config_path);++ tokio::runtime::Builder::new_multi_thread()+ .enable_all()+ .build()?+ .block_on(entrypoint::ampc::dht::run(config))?;+ }+ AmpcOptions::HarmonicWorker { config_path } => {+ let config: config::HarmonicWorkerConfig = load_toml_config(config_path);+ entrypoint::ampc::harmonic_centrality::worker::run(config)?;+ }+ AmpcOptions::HarmonicCoordinator { config_path } => {+ let config: config::HarmonicCoordinatorConfig = load_toml_config(config_path);+ entrypoint::ampc::harmonic_centrality::coordinator::run(config)?;+ }+ },}Ok(())diff --git a/crates/core/src/mapreduce/dht/client.rs b/crates/core/src/mapreduce/dht/client.rsdeleted file mode 100644index e7661b45..00000000--- a/crates/core/src/mapreduce/dht/client.rs+++ /dev/null@@ -1,135 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2024 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use rand::seq::SliceRandom;-use std::{collections::BTreeMap, net::SocketAddr};--use crate::{- distributed::{- cluster::Cluster,- member::{Service, ShardId},- },- Result,-};--use super::network::api;--struct Node {- api: api::RemoteClient,-}--impl Node {- fn new(addr: SocketAddr) -> Self {- let api = api::RemoteClient::new(addr);-- Self { api }- }-- async fn get(&self, key: Vec<u8>) -> Result<Option<Vec<u8>>> {- self.api.get(key).await- }-- async fn set(&self, key: Vec<u8>, value: Vec<u8>) -> Result<()> {- self.api.set(key, value).await- }-}--struct Shard {- nodes: Vec<Node>,-}--impl Shard {- fn new() -> Self {- Self { nodes: Vec::new() }- }-- fn add_node(&mut self, addr: SocketAddr) {- self.nodes.push(Node::new(addr));- }-- fn node(&self) -> &Node {- self.nodes.choose(&mut rand::thread_rng()).unwrap()- }-- async fn get(&self, key: Vec<u8>) -> Result<Option<Vec<u8>>> {- self.node().get(key).await- }-- async fn set(&self, key: Vec<u8>, value: Vec<u8>) -> Result<()> {- self.node().set(key, value).await- }-}--pub struct Client {- shards: BTreeMap<ShardId, Shard>,-}--impl Client {- pub async fn new(cluster: &Cluster) -> Self {- let dht_members = cluster- .members()- .await- .into_iter()- .filter_map(|member| {- if let Service::Dht { shard, host } = member.service {- Some((shard, host))- } else {- None- }- })- .collect::<Vec<_>>();-- let mut shards = BTreeMap::new();-- for (shard, host) in dht_members {- shards- .entry(shard)- .or_insert_with(Shard::new)- .add_node(host);- }-- Self { shards }- }-- pub fn add_node(&mut self, shard_id: ShardId, addr: SocketAddr) {- self.shards- .entry(shard_id)- .or_insert_with(Shard::new)- .add_node(addr);- }-- fn shard_for_key(&self, key: &[u8]) -> Result<&Shard> {- let ids = self.shards.keys().collect::<Vec<_>>();-- if ids.is_empty() {- return Err(anyhow::anyhow!("No shards"));- }-- let hash = md5::compute(key);- let hash = u64::from_le_bytes((&hash.0[..(u64::BITS / 8) as usize]).try_into().unwrap());-- let shard_id = ids[hash as usize % ids.len()];- Ok(self.shards.get(shard_id).unwrap())- }-- pub async fn get(&self, key: Vec<u8>) -> Result<Option<Vec<u8>>> {- self.shard_for_key(&key)?.get(key).await- }-- pub async fn set(&self, key: Vec<u8>, value: Vec<u8>) -> Result<()> {- self.shard_for_key(&key)?.set(key, value).await- }-}diff --git a/crates/core/src/mapreduce/dht/network/api.rs b/crates/core/src/mapreduce/dht/network/api.rsdeleted file mode 100644index e4a111b6..00000000--- a/crates/core/src/mapreduce/dht/network/api.rs+++ /dev/null@@ -1,183 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2024 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>--use std::{net::SocketAddr, time::Duration};--use crate::{distributed::retry_strategy::RandomBackoff, Result};-use anyhow::anyhow;-use openraft::{- error::{ClientWriteError, ForwardToLeader, RaftError},- BasicNode,-};-use tokio::sync::RwLock;--use crate::{distributed::sonic, mapreduce::dht::NodeId};--use super::Server;--#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]-pub struct Set {- pub key: Vec<u8>,- pub value: Vec<u8>,-}--#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]-pub struct Get {- pub key: Vec<u8>,-}--impl sonic::service::Message<Server> for Set {- type Response = Result<(), RaftError<NodeId, ClientWriteError<NodeId, BasicNode>>>;-- async fn handle(self, server: &Server) -> Self::Response {- tracing::debug!("received set request: {:?}", self);-- match server.raft.client_write(self.into()).await {- Ok(_) => Ok(()),- Err(e) => Err(e),- }- }-}--impl sonic::service::Message<Server> for Get {- type Response = Option<Vec<u8>>;-- async fn handle(self, server: &Server) -> Self::Response {- server- .state_machine_store- .state_machine- .read()- .await- .data- .get(&self.key)- .cloned()- }-}--pub struct RemoteClient {- self_remote: sonic::replication::RemoteClient<Server>,- likely_leader: RwLock<sonic::replication::RemoteClient<Server>>,-}--impl RemoteClient {- pub fn new(addr: SocketAddr) -> Self {- Self {- self_remote: sonic::replication::RemoteClient::new(addr),- likely_leader: RwLock::new(sonic::replication::RemoteClient::new(addr)),- }- }-- fn retry_strat() -> impl Iterator<Item = std::time::Duration> {- RandomBackoff::new(- std::time::Duration::from_millis(200),- std::time::Duration::from_secs(1),- )- }-- pub async fn set(&self, key: Vec<u8>, value: Vec<u8>) -> Result<()> {- for backoff in Self::retry_strat() {- let res = self- .likely_leader- .read()- .await- .send_with_timeout(- &Set {- key: key.clone(),- value: value.clone(),- },- Duration::from_secs(5),- )- .await;-- tracing::debug!(".set() got response: {res:?}");-- match res {- Ok(res) => match res {- Ok(_) => return Ok(()),- Err(RaftError::APIError(e)) => match e {- ClientWriteError::ForwardToLeader(ForwardToLeader {- leader_id: _,- leader_node,- }) => match leader_node {- Some(leader_node) => {- let mut likely_leader = self.likely_leader.write().await;- *likely_leader = sonic::replication::RemoteClient::new(- leader_node- .addr- .parse()- .expect("node addr should always be valid addr"),- );- }- None => {- tokio::time::sleep(backoff).await;- }- },- ClientWriteError::ChangeMembershipError(_) => {- unreachable!(".set() should not change membership")- }- },- Err(RaftError::Fatal(e)) => return Err(e.into()),- },- Err(e) => match e {- sonic::Error::IO(_)- | sonic::Error::Serialization(_)- | sonic::Error::ConnectionTimeout- | sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {- tokio::time::sleep(backoff).await;- }- sonic::Error::BadRequest- | sonic::Error::BodyTooLarge {- body_size: _,- max_size: _,- }- | sonic::Error::Application(_) => return Err(e.into()),- },- }- }-- Err(anyhow!("failed to set key"))- }-- pub async fn get(&self, key: Vec<u8>) -> Result<Option<Vec<u8>>> {- for backoff in Self::retry_strat() {- match self- .self_remote- .send_with_timeout(&Get { key: key.clone() }, Duration::from_secs(5))- .await- {- Ok(res) => return Ok(res),- Err(e) => match e {- sonic::Error::IO(_)- | sonic::Error::Serialization(_)- | sonic::Error::ConnectionTimeout- | sonic::Error::RequestTimeout- | sonic::Error::PoolCreation => {- tokio::time::sleep(backoff).await;- }- sonic::Error::BadRequest- | sonic::Error::BodyTooLarge {- body_size: _,- max_size: _,- }- | sonic::Error::Application(_) => return Err(e.into()),- },- }- }-- Err(anyhow!("failed to get key"))- }-}diff --git a/crates/core/src/mapreduce/dht/network/mod.rs b/crates/core/src/mapreduce/dht/network/mod.rsdeleted file mode 100644index e3c1d01e..00000000--- a/crates/core/src/mapreduce/dht/network/mod.rs+++ /dev/null@@ -1,87 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2024 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>--pub mod api;-pub mod raft;--use api::{Get, Set};-use std::{collections::BTreeMap, net::SocketAddr, sync::Arc};--use openraft::{BasicNode, Raft, RaftNetworkFactory};--use crate::sonic_service;--use self::raft::RemoteClient;--use super::{store::StateMachineStore, NodeId, TypeConfig};--#[derive(Clone)]-pub struct Network;--impl RaftNetworkFactory<TypeConfig> for Network {- type Network = RemoteClient;-- async fn new_client(&mut self, target: NodeId, node: &BasicNode) -> Self::Network {- RemoteClient::new(target, node.clone())- }-}--pub type AppendEntries = openraft::raft::AppendEntriesRequest<TypeConfig>;-pub type AppendEntriesResponse = openraft::raft::AppendEntriesResponse<NodeId>;--pub type InstallSnapshot = openraft::raft::InstallSnapshotRequest<TypeConfig>;-pub type InstallSnapshotResponse = openraft::raft::InstallSnapshotResponse<NodeId>;--pub type Vote = openraft::raft::VoteRequest<NodeId>;-pub type VoteResponse = openraft::raft::VoteResponse<NodeId>;--#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]-pub struct AddLearner {- pub id: NodeId,- pub addr: SocketAddr,-}--#[derive(serde::Serialize, serde::Deserialize, Debug, Clone)]-pub struct AddNodes {- members: BTreeMap<NodeId, BasicNode>,-}--sonic_service!(- Server,- [- AppendEntries,- InstallSnapshot,- Vote,- AddLearner,- AddNodes,- Get,- Set,- ]-);--pub struct Server {- raft: Raft<TypeConfig>,- state_machine_store: Arc<StateMachineStore>,-}--impl Server {- pub fn new(raft: Raft<TypeConfig>, state_machine_store: Arc<StateMachineStore>) -> Self {- Self {- raft,- state_machine_store,- }- }-}diff --git a/crates/core/src/mapreduce/dht/store.rs b/crates/core/src/mapreduce/dht/store.rsdeleted file mode 100644index 6ad043b9..00000000--- a/crates/core/src/mapreduce/dht/store.rs+++ /dev/null@@ -1,266 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2024 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>-use std::collections::BTreeMap;-use std::fmt::Debug;-use std::io::Cursor;-use std::sync::Arc;-use std::sync::Mutex;--use openraft::storage::RaftStateMachine;-use openraft::storage::Snapshot;-use openraft::BasicNode;-use openraft::Entry;-use openraft::EntryPayload;-use openraft::LogId;-use openraft::RaftSnapshotBuilder;-use openraft::RaftTypeConfig;-use openraft::SnapshotMeta;-use openraft::StorageError;-use openraft::StorageIOError;-use openraft::StoredMembership;-use serde::Deserialize;-use serde::Serialize;-use tokio::sync::RwLock;--use crate::mapreduce::dht::network::api;--use super::NodeId;-use super::TypeConfig;-use super::{Request, Response};--#[derive(Debug)]-pub struct StoredSnapshot {- pub meta: SnapshotMeta<NodeId, BasicNode>,-- /// The data of the state machine at the time of this snapshot.- pub data: Vec<u8>,-}--#[derive(Serialize, Deserialize, Debug, Default, Clone)]-pub struct StateMachineData {- pub last_applied_log: Option<LogId<NodeId>>,- pub last_membership: StoredMembership<NodeId, BasicNode>,-- /// Application data.- pub data: BTreeMap<Vec<u8>, Vec<u8>>,-}--#[derive(Debug, Default)]-pub struct StateMachineStore {- pub state_machine: RwLock<StateMachineData>,- snapshot_idx: Arc<Mutex<u64>>,-- /// The last received snapshot.- current_snapshot: RwLock<Option<StoredSnapshot>>,-}--impl RaftSnapshotBuilder<TypeConfig> for Arc<StateMachineStore> {- #[tracing::instrument(level = "trace", skip(self))]- async fn build_snapshot(&mut self) -> Result<Snapshot<TypeConfig>, StorageError<NodeId>> {- let data;- let last_applied_log;- let last_membership;-- {- // Serialize the data of the state machine.- let state_machine = self.state_machine.read().await;- data = bincode::serialize(&*state_machine)- .map_err(|e| StorageIOError::read_state_machine(&e))?;-- last_applied_log = state_machine.last_applied_log;- last_membership = state_machine.last_membership.clone();- }-- let snapshot_idx = {- let mut l = self.snapshot_idx.lock().unwrap();- *l += 1;- *l- };-- let snapshot_id = if let Some(last) = last_applied_log {- format!("{}-{}-{}", last.leader_id, last.index, snapshot_idx)- } else {- format!("--{}", snapshot_idx)- };-- let meta = SnapshotMeta {- last_log_id: last_applied_log,- last_membership,- snapshot_id,- };-- let snapshot = StoredSnapshot {- meta: meta.clone(),- data: data.clone(),- };-- {- let mut current_snapshot = self.current_snapshot.write().await;- *current_snapshot = Some(snapshot);- }-- Ok(Snapshot {- meta,- snapshot: Box::new(Cursor::new(data)),- })- }-}--impl RaftStateMachine<TypeConfig> for Arc<StateMachineStore> {- type SnapshotBuilder = Self;-- async fn applied_state(- &mut self,- ) -> Result<(Option<LogId<NodeId>>, StoredMembership<NodeId, BasicNode>), StorageError<NodeId>>- {- let state_machine = self.state_machine.read().await;- Ok((- state_machine.last_applied_log,- state_machine.last_membership.clone(),- ))- }-- #[tracing::instrument(level = "trace", skip(self, entries))]- async fn apply<I>(&mut self, entries: I) -> Result<Vec<Response>, StorageError<NodeId>>- where- I: IntoIterator<Item = Entry<TypeConfig>> + Send,- {- let mut res = Vec::new();- let mut sm = self.state_machine.write().await;-- for entry in entries {- tracing::debug!(%entry.log_id, "replicate to sm");-- if let Some(ref last) = sm.last_applied_log {- if last >= &entry.log_id {- res.push(Response::Empty);- continue;- }- }-- sm.last_applied_log = Some(entry.log_id);-- match entry.payload {- EntryPayload::Blank => res.push(Response::Empty),- EntryPayload::Normal(ref req) => match req {- Request::Set(api::Set { key, value }) => {- sm.data.insert(key.clone(), value.clone());- res.push(Response::Set(Ok(())))- }-- Request::Get(api::Get { key: _ }) => {- unreachable!("Get requests should not be replicated")- }- },- EntryPayload::Membership(ref mem) => {- sm.last_membership = StoredMembership::new(Some(entry.log_id), mem.clone());- res.push(Response::Empty)- }- };- }- Ok(res)- }-- #[tracing::instrument(level = "trace", skip(self))]- async fn begin_receiving_snapshot(- &mut self,- ) -> Result<Box<<TypeConfig as RaftTypeConfig>::SnapshotData>, StorageError<NodeId>> {- Ok(Box::new(Cursor::new(Vec::new())))- }-- #[tracing::instrument(level = "trace", skip(self, snapshot))]- async fn install_snapshot(- &mut self,- meta: &SnapshotMeta<NodeId, BasicNode>,- snapshot: Box<<TypeConfig as RaftTypeConfig>::SnapshotData>,- ) -> Result<(), StorageError<NodeId>> {- tracing::info!(- { snapshot_size = snapshot.get_ref().len() },- "decoding snapshot for installation"- );-- let new_snapshot = StoredSnapshot {- meta: meta.clone(),- data: snapshot.into_inner(),- };-- // Update the state machine.- {- let data: BTreeMap<Vec<u8>, Vec<u8>> = bincode::deserialize(&new_snapshot.data)- .map_err(|e| {- StorageIOError::read_snapshot(Some(new_snapshot.meta.signature()), &e)- })?;-- let mut state_machine = self.state_machine.write().await;- state_machine.data = data;- state_machine.last_applied_log = meta.last_log_id;- state_machine.last_membership = meta.last_membership.clone();- }-- // Update current snapshot.- let mut current_snapshot = self.current_snapshot.write().await;- *current_snapshot = Some(new_snapshot);- Ok(())- }-- #[tracing::instrument(level = "trace", skip(self))]- async fn get_current_snapshot(- &mut self,- ) -> Result<Option<Snapshot<TypeConfig>>, StorageError<NodeId>> {- match &*self.current_snapshot.read().await {- Some(snapshot) => {- let data = snapshot.data.clone();- Ok(Some(Snapshot {- meta: snapshot.meta.clone(),- snapshot: Box::new(Cursor::new(data)),- }))- }- None => Ok(None),- }- }-- async fn get_snapshot_builder(&mut self) -> Self::SnapshotBuilder {- Arc::clone(self)- }-}--#[cfg(test)]-mod tests {- use openraft::testing::{StoreBuilder, Suite};-- type LogStore = crate::mapreduce::dht::log_store::LogStore<TypeConfig>;-- use super::*;-- struct MemStoreBuilder {}-- impl StoreBuilder<TypeConfig, LogStore, Arc<StateMachineStore>, ()> for MemStoreBuilder {- async fn build(- &self,- ) -> Result<((), LogStore, Arc<StateMachineStore>), StorageError<NodeId>> {- let log_store = LogStore::default();- let sm = Arc::new(StateMachineStore::default());-- Ok(((), log_store, sm))- }- }-- #[test]- pub fn test_raft_impl() -> Result<(), StorageError<NodeId>> {- Suite::test_all(MemStoreBuilder {})?;- Ok(())- }-}diff --git a/crates/core/src/mapreduce/manager.rs b/crates/core/src/mapreduce/manager.rsdeleted file mode 100644index 2ebb3326..00000000--- a/crates/core/src/mapreduce/manager.rs+++ /dev/null@@ -1,286 +0,0 @@-use super::{Error, MapReduceConnection, Result, Worker};-use super::{Map, Reduce};-use crate::distributed::retry_strategy::ExponentialBackoff;-use crate::mapreduce::Task;-use futures::StreamExt;-use itertools::Itertools;-use serde::de::DeserializeOwned;-use serde::Serialize;-use std::net::SocketAddr;-use std::net::ToSocketAddrs;-use std::ops::Deref;-use std::sync::atomic::{AtomicU32, Ordering};-use std::sync::Arc;-use std::time::Duration;-use tokio::sync::Mutex;-use tracing::{debug, warn};--#[derive(Debug)]-struct RemoteWorker {- addr: SocketAddr,-}--impl RemoteWorker {- fn retry_strategy() -> impl Iterator<Item = Duration> {- ExponentialBackoff::from_millis(10)- .with_limit(Duration::from_secs(100))- .take(15)- }-- async fn connect<W, I, O>(&self) -> Result<MapReduceConnection<I, O>>- where- W: Worker,- I: Map<W, O> + Send,- O: Serialize + DeserializeOwned + Send,- {- for dur in RemoteWorker::retry_strategy() {- if let Ok(conn) = MapReduceConnection::create(self.addr).await {- debug!("connected");- return Ok(conn);- }-- std::thread::sleep(dur);- }-- Err(Error::NoResponse)- }-- async fn perform<W, I, O>(&self, job: I) -> Result<O>- where- W: Worker,- I: Map<W, O> + Send,- O: Serialize + DeserializeOwned + Send,- {- let conn = self.connect::<W, I, O>().await?;- match conn.send(&Task::Job(job)).await {- Ok(Some(res)) => Ok(res),- _ => Err(Error::NoResponse),- }- }-- async fn stop<W, I, O>(&self) -> Result<()>- where- W: Worker,- I: Map<W, O> + Send,- O: Serialize + DeserializeOwned + Send,- {- debug!("closing worker {:}", self.addr);- let conn = self.connect().await?;- let res = conn.send(&Task::<I>::AllFinished).await?;-- debug_assert!(res.is_none());-- Ok(())- }-}--struct WorkerGuard<'a> {- from_pool: &'a WorkerPool,- worker: Arc<RemoteWorker>,-}--impl<'a> WorkerGuard<'a> {- fn new(pool: &'a WorkerPool, worker: Arc<RemoteWorker>) -> Self {- Self {- worker,- from_pool: pool,- }- }-- async fn success(self) {- self.from_pool.insert(Arc::clone(&self.worker)).await;- }-}--impl<'a> Deref for WorkerGuard<'a> {- type Target = Arc<RemoteWorker>;-- fn deref(&self) -> &Self::Target {- &self.worker- }-}--impl<'a> Drop for WorkerGuard<'a> {- fn drop(&mut self) {- self.from_pool.put_back();- }-}--struct WorkerPool {- all_workers: Vec<Arc<RemoteWorker>>,- ready_workers: Mutex<Vec<Arc<RemoteWorker>>>,- running_workers: AtomicU32,-}--impl WorkerPool {- fn new<A>(workers: &[A]) -> Self- where- A: ToSocketAddrs + std::fmt::Debug,- {- let all_workers: Vec<Arc<RemoteWorker>> = workers- .iter()- .flat_map(|addr| {- addr.to_socket_addrs().unwrap_or_else(|_| {- panic!("failed to transform {addr:?} into a socket address")- })- })- .map(|addr| Arc::new(RemoteWorker { addr }))- .collect();-- Self {- ready_workers: Mutex::new(all_workers.clone()),- all_workers,- running_workers: AtomicU32::new(0),- }- }-- fn put_back(&self) {- self.running_workers.fetch_sub(1, Ordering::SeqCst);- }-- async fn insert(&self, worker: Arc<RemoteWorker>) {- self.ready_workers.lock().await.push(worker);- }-- async fn get_worker(&self) -> Result<Option<WorkerGuard<'_>>> {- let mut ready_workers = self.ready_workers.lock().await;- if ready_workers.len() as u32 + self.running_workers.load(Ordering::SeqCst) == 0 {- return Err(Error::NoAvailableWorker);- }-- if let Some(worker) = ready_workers.pop() {- self.running_workers.fetch_add(1, Ordering::SeqCst);- Ok(Some(WorkerGuard::new(self, worker)))- } else {- Ok(None)- }- }-- async fn stop_workers<W, I, O>(&self)- where- W: Worker,- I: Map<W, O> + Send,- O: Serialize + DeserializeOwned + Send,- {- let mut failing_workers = Vec::new();- for worker in &self.all_workers {- if worker.stop::<W, I, O>().await.is_err() {- failing_workers.push(worker);- }- }-- if !failing_workers.is_empty() {- debug!(- "failed to stop the following workers: {:#?}",- failing_workers- );- }- }-- fn size(&self) -> usize {- self.all_workers.len()- }-}--pub struct Manager {- pool: WorkerPool,-}--impl Manager {- pub fn new<A>(workers: &[A]) -> Self- where- A: ToSocketAddrs + std::fmt::Debug,- {- Self {- pool: WorkerPool::new(workers),- }- }-- async fn try_map<W, I, O>(&self, job: &I) -> Result<O>- where- W: Worker,- I: Map<W, O> + Send + Clone,- O: Serialize + DeserializeOwned + Send,- {- loop {- match self.pool.get_worker().await? {- Some(worker) => {- let res = worker.perform(job.clone()).await?;- worker.success().await;-- return Ok(res);- }- None => std::thread::sleep(std::time::Duration::from_millis(1000)),- }- }- }-- /// Execute job on one of the remote machines. If the remote machine fails for some reason,- /// the job should be allocated to another machine.- pub async fn map<W, I, O>(&self, job: I) -> O- where- W: Worker,- I: Map<W, O> + Send + Clone,- O: Serialize + DeserializeOwned + Send,- {- loop {- match self.try_map(&job).await {- Ok(res) => return res,- Err(Error::NoAvailableWorker) => panic!("{}", Error::NoAvailableWorker),- Err(err) => {- warn!("Worker failed - rescheduling job");- debug!("{:?}", err);- }- }- }- }-- fn reduce<O1, O2>(acc: Option<O2>, elem: O1) -> O2- where- O1: Serialize + DeserializeOwned + Send,- O2: From<O1> + Reduce<O1> + Send,- {- match acc {- Some(acc) => acc.reduce(elem),- None => elem.into(),- }- }-- #[allow(clippy::trait_duplication_in_bounds)]- async fn get_results<W, I, O1, O2>(&self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>- where- W: Worker,- I: Map<W, O1> + Send + Clone,- O1: Serialize + DeserializeOwned + Send,- O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,- {- let mut acc = None;-- for chunk in jobs.chunks(self.pool.size()).into_iter() {- let results = futures::stream::iter(chunk.map(|job| self.map::<W, I, O1>(job)))- .buffer_unordered(self.pool.size())- .collect::<Vec<_>>()- .await;-- for elem in results {- acc = Some(Self::reduce(acc, elem));- }- }-- acc- }-- #[allow(clippy::trait_duplication_in_bounds)]- pub async fn run<W, I, O1, O2>(self, jobs: impl Iterator<Item = I> + Send) -> Option<O2>- where- W: Worker,- I: Map<W, O1> + Send + Clone,- O1: Serialize + DeserializeOwned + Send,- O2: From<O1> + Reduce<O1> + Send + Reduce<O2>,- {- let result = self.get_results(jobs).await;- self.pool.stop_workers::<W, I, O1>().await;-- result- }-}diff --git a/crates/core/src/mapreduce/mod.rs b/crates/core/src/mapreduce/mod.rsdeleted file mode 100644index a96d1e18..00000000--- a/crates/core/src/mapreduce/mod.rs+++ /dev/null@@ -1,68 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--//! MapReduce is a distributed computing framework for processing large data sets across-//! clusters of computers using a master node to coordinate the work.--use serde::{de::DeserializeOwned, Deserialize, Serialize};--pub mod dht;-mod manager;-mod worker;--pub use manager::Manager;-use thiserror::Error;-pub use worker::StatelessWorker;-pub use worker::Worker;--use crate::distributed::sonic;--pub(crate) type Result<T> = std::result::Result<T, Error>;--#[derive(Error, Debug)]-pub enum Error {- #[error("network error")]- Sonic(#[from] sonic::Error),-- #[error("could not get a working worker")]- NoAvailableWorker,-- #[error("did not get a reponse")]- NoResponse,-}--pub trait Map<W, T>-where- Self: Serialize + DeserializeOwned + Send,- T: Serialize + DeserializeOwned + Send,- W: Worker,-{- fn map(&self, worker: &W) -> T;-}--pub trait Reduce<T> {- #[must_use]- fn reduce(self, element: T) -> Self;-}--#[derive(Serialize, Deserialize, Debug)]-enum Task<T> {- Job(T),- AllFinished,-}--type MapReduceServer<I, O> = sonic::Server<Task<I>, Option<O>>;-type MapReduceConnection<I, O> = sonic::Connection<Task<I>, Option<O>>;diff --git a/crates/core/src/mapreduce/worker.rs b/crates/core/src/mapreduce/worker.rsdeleted file mode 100644index adf34d0b..00000000--- a/crates/core/src/mapreduce/worker.rs+++ /dev/null@@ -1,60 +0,0 @@-// Stract is an open source web search engine.-// Copyright (C) 2023 Stract ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::{future::Future, net::SocketAddr};--use crate::mapreduce::MapReduceServer;--use super::{Map, Result, Task};-use serde::{de::DeserializeOwned, Serialize};-use tracing::{debug, info};--#[derive(Default)]-pub struct StatelessWorker {}--pub trait Worker {- fn run<I, O>(&self, addr: SocketAddr) -> impl Future<Output = Result<()>>- where- Self: Sized,- I: Map<Self, O> + Send + Sync,- O: Serialize + DeserializeOwned + Send + Sync,- {- async move {- let server = MapReduceServer::<I, O>::bind(addr).await?;- info!("worker listening on: {:}", addr);-- loop {- let req = server.accept().await?;- debug!("received request");- match req.body() {- Task::Job(job) => {- debug!("request is a job");- let res = job.map(self);- req.respond(Some(res)).await?;- }- Task::AllFinished => {- req.respond(None).await?;- break;- }- }- }-- Ok(())- }- }-}--impl Worker for StatelessWorker {}diff --git a/crates/core/src/naive_bayes.rs b/crates/core/src/naive_bayes.rsindex 729019ba..cf7e1c8c 100644--- a/crates/core/src/naive_bayes.rs+++ b/crates/core/src/naive_bayes.rs@@ -15,10 +15,13 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use crate::intmap::IntMap;-use hashbrown::{HashMap, HashSet};+use ahash::{HashMapExt, HashSetExt};use itertools::Itertools;use std::hash::Hash;+type HashMap<K, V> = std::collections::HashMap<K, V, ahash::RandomState>;+type HashSet<T> = std::collections::HashSet<T, ahash::RandomState>;+/// Laplace smoothing factorconst ALPHA: f32 = 1.0;@@ -30,7 +33,7 @@ pub struct TfIdf {}/// Vectorizer to convert text into TF-IDF features-#[derive(serde::Deserialize, serde::Serialize)]+#[derive(bincode::Encode, bincode::Decode)]struct TfidfVectorizer {vocabulary: HashMap<String, u64>,idf: IntMap<u64, f32>,@@ -127,7 +130,7 @@ pub struct Prediction<L> {}/// Naive Bayes Classifier-#[derive(serde::Deserialize, serde::Serialize)]+#[derive(bincode::Encode, bincode::Decode)]pub struct NaiveBayes<L: Label> {classes: Vec<L>,class_prior: Vec<f32>,@@ -260,7 +263,7 @@ impl<L: Label> NaiveBayes<L> {}}-#[derive(serde::Deserialize, serde::Serialize)]+#[derive(bincode::Encode, bincode::Decode)]pub struct Pipeline<L: Label> {vectorizer: TfidfVectorizer,classifier: NaiveBayes<L>,diff --git a/crates/core/src/prehashed.rs b/crates/core/src/prehashed.rsindex 4453d9c9..7c39eb9b 100644--- a/crates/core/src/prehashed.rs+++ b/crates/core/src/prehashed.rs@@ -16,9 +16,9 @@use std::hash::Hash;-use serde::{Deserialize, Serialize};--#[derive(Debug, Eq, Clone, Copy, Serialize, Deserialize)]+#[derive(+ Debug, Eq, Clone, Copy, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,+)]pub struct Prehashed(pub u128);impl From<u128> for Prehashed {diff --git a/crates/core/src/ranking/bitvec_similarity.rs b/crates/core/src/ranking/bitvec_similarity.rsindex af1d8e01..900c8986 100644--- a/crates/core/src/ranking/bitvec_similarity.rs+++ b/crates/core/src/ranking/bitvec_similarity.rs@@ -15,9 +15,8 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use itertools::Itertools;-use serde::{Deserialize, Serialize};-#[derive(Serialize, Deserialize, Debug, Clone)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]struct VeryJankyBloomFilter {data: Vec<u64>,ones: usize,@@ -66,7 +65,7 @@ impl VeryJankyBloomFilter {}}-#[derive(Serialize, Deserialize, Debug, Clone)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]struct Posting {ranks: Vec<u64>,}@@ -105,7 +104,7 @@ impl Posting {}}-#[derive(Serialize, Deserialize, Debug, Clone)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug, Clone)]pub struct BitVec {bloom: VeryJankyBloomFilter,posting: Posting,diff --git a/crates/core/src/ranking/bm25.rs b/crates/core/src/ranking/bm25.rsindex e9649da6..26e967ee 100644--- a/crates/core/src/ranking/bm25.rs+++ b/crates/core/src/ranking/bm25.rs@@ -1,7 +1,6 @@// source: https://github.com/quickwit-oss/tantivy/blob/main/src/query/bm25.rsuse itertools::Itertools;-use serde::{Deserialize, Serialize};use tantivy::fieldnorm::FieldNormReader;use tantivy::query::Explanation;@@ -29,7 +28,9 @@ fn compute_tf_cache(average_fieldnorm: Score) -> [Score; 256] {cache}-#[derive(Clone, PartialEq, Debug, Serialize, Deserialize)]+#[derive(+ Clone, PartialEq, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,+)]pub struct Bm25Params {pub idf: Score,pub avg_fieldnorm: Score,diff --git a/crates/core/src/ranking/inbound_similarity.rs b/crates/core/src/ranking/inbound_similarity.rsindex 1f54f75d..5dd2b374 100644--- a/crates/core/src/ranking/inbound_similarity.rs+++ b/crates/core/src/ranking/inbound_similarity.rs@@ -16,7 +16,7 @@use std::{fs::File,- io::{BufReader, BufWriter, Read},+ io::{BufReader, BufWriter},path::Path,sync::Arc,};@@ -26,7 +26,6 @@ use fnv::FnvHashMap as HashMap;use fnv::FnvHashSet as HashSet;use indicatif::ParallelProgressIterator;use rayon::prelude::*;-use serde::{Deserialize, Serialize};use crate::{webgraph::{NodeID, Webgraph},@@ -164,7 +163,7 @@ impl Scorer {}}-#[derive(Serialize, Deserialize, Default)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Default)]struct PreCalculatedSimilarities {map: HashMap<NodeID, HashMap<NodeID, f64>>,}@@ -179,7 +178,7 @@ impl PreCalculatedSimilarities {}}-#[derive(Serialize, Deserialize, Default)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Default)]struct VecMap {map: HashMap<NodeID, bitvec_similarity::BitVec>,}@@ -213,7 +212,7 @@ impl VecMap {}}-#[derive(Serialize, Deserialize, Default)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Default)]pub struct InboundSimilarity {vectors: Arc<VecMap>,precalculated: Arc<PreCalculatedSimilarities>,@@ -312,7 +311,7 @@ impl InboundSimilarity {.open(path)?,);- bincode::serialize_into(&mut file, &self)?;+ bincode::encode_into_std_write(self, &mut file, bincode::config::standard())?;Ok(())}@@ -325,10 +324,10 @@ impl InboundSimilarity {let file = File::open(path)?;let mut reader = BufReader::new(file);- let mut buf = Vec::new();- reader.read_to_end(&mut buf)?;-- Ok(bincode::deserialize(&buf)?)+ Ok(bincode::decode_from_std_read(+ &mut reader,+ bincode::config::standard(),+ )?)}pub fn knows_about(&self, node_id: NodeID) -> bool {diff --git a/crates/core/src/ranking/initial.rs b/crates/core/src/ranking/initial.rsindex 5aa764de..8053df9e 100644--- a/crates/core/src/ranking/initial.rs+++ b/crates/core/src/ranking/initial.rs@@ -16,7 +16,7 @@use crate::fastfield_reader::FastFieldReader;use chrono::Utc;-use serde::{Deserialize, Serialize};+use tantivy::collector::{ScoreSegmentTweaker, ScoreTweaker};use tantivy::{DocId, SegmentReader};@@ -69,7 +69,9 @@ pub struct InitialSegmentScoreTweaker {computer: SignalComputer,}-#[derive(Debug, Serialize, Deserialize, Clone, PartialEq)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, PartialEq,+)]pub struct Score {pub total: f64,}diff --git a/crates/core/src/ranking/models/linear.rs b/crates/core/src/ranking/models/linear.rsindex c2abad17..0e5c63e6 100644--- a/crates/core/src/ranking/models/linear.rs+++ b/crates/core/src/ranking/models/linear.rs@@ -22,7 +22,7 @@ use crate::enum_map::EnumMap;use crate::ranking::SignalEnum;use crate::Result;-#[derive(serde::Serialize, serde::Deserialize, Debug)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug)]struct SerialziedLinearRegression {weights: HashMap<SignalEnum, f64>,}@@ -39,7 +39,7 @@ impl From<SerialziedLinearRegression> for LinearRegression {}}-#[derive(serde::Serialize, serde::Deserialize, Debug)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug)]pub struct LinearRegression {pub weights: EnumMap<SignalEnum, f64>,}diff --git a/crates/core/src/ranking/pipeline/stages/precision.rs b/crates/core/src/ranking/pipeline/stages/precision.rsindex 5d9f011d..34af8f26 100644--- a/crates/core/src/ranking/pipeline/stages/precision.rs+++ b/crates/core/src/ranking/pipeline/stages/precision.rs@@ -16,8 +16,6 @@use std::sync::Arc;-use serde::{Deserialize, Serialize};-use crate::{collector,config::CollectorConfig,@@ -35,7 +33,7 @@ use crate::{use super::RecallRankingWebpage;-#[derive(Clone, Debug, Serialize, Deserialize)]+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct PrecisionRankingWebpage {pub retrieved_webpage: RetrievedWebpage,pub ranking: RecallRankingWebpage,diff --git a/crates/core/src/ranking/pipeline/stages/recall.rs b/crates/core/src/ranking/pipeline/stages/recall.rsindex e776f98f..ed06a9ad 100644--- a/crates/core/src/ranking/pipeline/stages/recall.rs+++ b/crates/core/src/ranking/pipeline/stages/recall.rs@@ -16,8 +16,6 @@use std::sync::Arc;-use serde::{Deserialize, Serialize};-use crate::{collector,config::CollectorConfig,@@ -34,7 +32,7 @@ use crate::{searcher::SearchQuery,};-#[derive(Clone, Debug, Serialize, Deserialize)]+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct StoredEmbeddings(Vec<u8>);impl StoredEmbeddings {@@ -43,7 +41,7 @@ impl StoredEmbeddings {}}-#[derive(Clone, Debug, Serialize, Deserialize)]+#[derive(Clone, Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct RecallRankingWebpage {pub pointer: WebpagePointer,pub signals: EnumMap<SignalEnum, SignalScore>,diff --git a/crates/core/src/ranking/signal/mod.rs b/crates/core/src/ranking/signal/mod.rsindex f1954a37..914bfcb5 100644--- a/crates/core/src/ranking/signal/mod.rs+++ b/crates/core/src/ranking/signal/mod.rs@@ -17,7 +17,7 @@use crate::enum_map::EnumMap;use optics::ast::RankingTarget;use optics::Optic;-use serde::{Deserialize, Serialize};+use std::str::FromStr;use thiserror::Error;@@ -102,7 +102,16 @@ pub struct ComputedSignal {pub score: SignalScore,}-#[derive(Debug, Clone, Copy, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug,+ Clone,+ Copy,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct SignalScore {pub coefficient: f64,diff --git a/crates/core/src/ranking/signal/non_text.rs b/crates/core/src/ranking/signal/non_text.rsindex 20de67f8..cf3b2144 100644--- a/crates/core/src/ranking/signal/non_text.rs+++ b/crates/core/src/ranking/signal/non_text.rs@@ -101,7 +101,18 @@ fn host_id(fastfield_reader: &FieldReader<'_>) -> Option<NodeID> {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct HostCentrality;impl Signal for HostCentrality {fn default_coefficient(&self) -> f64 {@@ -128,7 +139,18 @@ impl Signal for HostCentrality {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct HostCentralityRank;impl Signal for HostCentralityRank {fn default_coefficient(&self) -> f64 {@@ -155,7 +177,18 @@ impl Signal for HostCentralityRank {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct PageCentrality;impl Signal for PageCentrality {fn default_coefficient(&self) -> f64 {@@ -182,7 +215,18 @@ impl Signal for PageCentrality {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct PageCentralityRank;impl Signal for PageCentralityRank {fn default_coefficient(&self) -> f64 {@@ -209,7 +253,18 @@ impl Signal for PageCentralityRank {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct IsHomepage;impl Signal for IsHomepage {fn default_coefficient(&self) -> f64 {@@ -236,7 +291,18 @@ impl Signal for IsHomepage {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct FetchTimeMs;impl Signal for FetchTimeMs {fn default_coefficient(&self) -> f64 {@@ -273,7 +339,18 @@ impl Signal for FetchTimeMs {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct UpdateTimestamp;impl Signal for UpdateTimestamp {fn default_coefficient(&self) -> f64 {@@ -307,7 +384,18 @@ impl Signal for UpdateTimestamp {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct TrackerScore;impl Signal for TrackerScore {fn default_coefficient(&self) -> f64 {@@ -335,7 +423,18 @@ impl Signal for TrackerScore {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Region;impl Signal for Region {fn default_coefficient(&self) -> f64 {@@ -365,7 +464,18 @@ impl Signal for Region {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct QueryCentrality;impl Signal for QueryCentrality {fn default_coefficient(&self) -> f64 {@@ -385,7 +495,18 @@ impl Signal for QueryCentrality {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct InboundSimilarity;impl Signal for InboundSimilarity {fn default_coefficient(&self) -> f64 {@@ -405,7 +526,18 @@ impl Signal for InboundSimilarity {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct LambdaMart;impl Signal for LambdaMart {fn default_coefficient(&self) -> f64 {@@ -421,7 +553,18 @@ impl Signal for LambdaMart {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct UrlDigits;impl Signal for UrlDigits {fn default_coefficient(&self) -> f64 {@@ -466,7 +609,18 @@ impl Signal for UrlDigits {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct UrlSlashes;impl Signal for UrlSlashes {fn default_coefficient(&self) -> f64 {@@ -502,7 +656,18 @@ impl Signal for UrlSlashes {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct LinkDensity;impl Signal for LinkDensity {fn default_coefficient(&self) -> f64 {diff --git a/crates/core/src/ranking/signal/prelude.rs b/crates/core/src/ranking/signal/prelude.rsindex 20cf8a8d..85c13e78 100644--- a/crates/core/src/ranking/signal/prelude.rs+++ b/crates/core/src/ranking/signal/prelude.rs@@ -1,3 +1,4 @@+use crate::enum_dispatch_from_discriminant;// Stract is an open source web search engine.// Copyright (C) 2024 Stract ApS//@@ -15,7 +16,6 @@// along with this program. If not, see <https://www.gnu.org/licenses/>use crate::enum_map::InsertEnumMapKey;-use crate::enum_dispatch_from_discriminant;use crate::schema::Field;use crate::{schema::{FastFieldEnum, TextFieldEnum},@@ -53,9 +53,26 @@ pub trait Signal:#[enum_dispatch(Signal)]#[derive(- Debug, serde::Serialize, serde::Deserialize, Clone, Copy, PartialEq, Eq, Hash, EnumDiscriminants,+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ EnumDiscriminants,)]-#[strum_discriminants(derive(VariantArray, serde::Serialize, serde::Deserialize, Hash))]+#[strum_discriminants(derive(+ VariantArray,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Hash+))]#[strum_discriminants(serde(rename_all = "snake_case"))]pub enum SignalEnum {Bm25Title,diff --git a/crates/core/src/ranking/signal/text.rs b/crates/core/src/ranking/signal/text.rsindex 6d247486..282d38b3 100644--- a/crates/core/src/ranking/signal/text.rs+++ b/crates/core/src/ranking/signal/text.rs@@ -61,7 +61,18 @@ fn idf_sum(field: &mut TextFieldData, doc: DocId) -> f64 {.sum::<f32>() as f64}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Bm25Title;impl Signal for Bm25Title {fn default_coefficient(&self) -> f64 {@@ -82,7 +93,18 @@ impl Signal for Bm25Title {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Bm25TitleBigrams;impl Signal for Bm25TitleBigrams {fn default_coefficient(&self) -> f64 {@@ -103,7 +125,18 @@ impl Signal for Bm25TitleBigrams {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Bm25TitleTrigrams;impl Signal for Bm25TitleTrigrams {fn default_coefficient(&self) -> f64 {@@ -124,7 +157,18 @@ impl Signal for Bm25TitleTrigrams {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Bm25CleanBody;impl Signal for Bm25CleanBody {fn default_coefficient(&self) -> f64 {@@ -145,7 +189,18 @@ impl Signal for Bm25CleanBody {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Bm25CleanBodyBigrams;impl Signal for Bm25CleanBodyBigrams {fn default_coefficient(&self) -> f64 {@@ -166,7 +221,18 @@ impl Signal for Bm25CleanBodyBigrams {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Bm25CleanBodyTrigrams;impl Signal for Bm25CleanBodyTrigrams {fn default_coefficient(&self) -> f64 {@@ -187,7 +253,18 @@ impl Signal for Bm25CleanBodyTrigrams {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Bm25StemmedTitle;impl Signal for Bm25StemmedTitle {fn default_coefficient(&self) -> f64 {@@ -208,7 +285,18 @@ impl Signal for Bm25StemmedTitle {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Bm25StemmedCleanBody;impl Signal for Bm25StemmedCleanBody {fn default_coefficient(&self) -> f64 {@@ -229,7 +317,18 @@ impl Signal for Bm25StemmedCleanBody {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Bm25AllBody;impl Signal for Bm25AllBody {fn default_coefficient(&self) -> f64 {@@ -250,7 +349,18 @@ impl Signal for Bm25AllBody {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Bm25Keywords;impl Signal for Bm25Keywords {fn default_coefficient(&self) -> f64 {@@ -271,7 +381,18 @@ impl Signal for Bm25Keywords {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Bm25BacklinkText;impl Signal for Bm25BacklinkText {fn default_coefficient(&self) -> f64 {@@ -292,7 +413,18 @@ impl Signal for Bm25BacklinkText {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct IdfSumUrl;impl Signal for IdfSumUrl {fn default_coefficient(&self) -> f64 {@@ -313,7 +445,18 @@ impl Signal for IdfSumUrl {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct IdfSumSite;impl Signal for IdfSumSite {fn default_coefficient(&self) -> f64 {@@ -334,7 +477,18 @@ impl Signal for IdfSumSite {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct IdfSumDomain;impl Signal for IdfSumDomain {fn default_coefficient(&self) -> f64 {@@ -355,7 +509,18 @@ impl Signal for IdfSumDomain {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct IdfSumSiteNoTokenizer;impl Signal for IdfSumSiteNoTokenizer {fn default_coefficient(&self) -> f64 {@@ -376,7 +541,18 @@ impl Signal for IdfSumSiteNoTokenizer {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct IdfSumDomainNoTokenizer;impl Signal for IdfSumDomainNoTokenizer {fn default_coefficient(&self) -> f64 {@@ -397,7 +573,18 @@ impl Signal for IdfSumDomainNoTokenizer {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct IdfSumDomainNameNoTokenizer;impl Signal for IdfSumDomainNameNoTokenizer {fn default_coefficient(&self) -> f64 {@@ -420,7 +607,18 @@ impl Signal for IdfSumDomainNameNoTokenizer {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct IdfSumDomainIfHomepage;impl Signal for IdfSumDomainIfHomepage {fn default_coefficient(&self) -> f64 {@@ -441,7 +639,18 @@ impl Signal for IdfSumDomainIfHomepage {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct IdfSumDomainNameIfHomepageNoTokenizer;impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {fn default_coefficient(&self) -> f64 {@@ -464,7 +673,18 @@ impl Signal for IdfSumDomainNameIfHomepageNoTokenizer {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct IdfSumDomainIfHomepageNoTokenizer;impl Signal for IdfSumDomainIfHomepageNoTokenizer {fn default_coefficient(&self) -> f64 {@@ -487,7 +707,18 @@ impl Signal for IdfSumDomainIfHomepageNoTokenizer {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct IdfSumTitleIfHomepage;impl Signal for IdfSumTitleIfHomepage {fn default_coefficient(&self) -> f64 {@@ -508,7 +739,18 @@ impl Signal for IdfSumTitleIfHomepage {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct CrossEncoderSnippet;impl Signal for CrossEncoderSnippet {fn default_coefficient(&self) -> f64 {@@ -524,7 +766,18 @@ impl Signal for CrossEncoderSnippet {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct CrossEncoderTitle;impl Signal for CrossEncoderTitle {fn default_coefficient(&self) -> f64 {@@ -540,7 +793,18 @@ impl Signal for CrossEncoderTitle {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct TitleEmbeddingSimilarity;impl Signal for TitleEmbeddingSimilarity {fn default_coefficient(&self) -> f64 {@@ -556,7 +820,18 @@ impl Signal for TitleEmbeddingSimilarity {}}-#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct KeywordEmbeddingSimilarity;impl Signal for KeywordEmbeddingSimilarity {fn default_coefficient(&self) -> f64 {diff --git a/crates/core/src/search_prettifier/entity.rs b/crates/core/src/search_prettifier/entity.rsindex 1d0e310c..b26d16f7 100644--- a/crates/core/src/search_prettifier/entity.rs+++ b/crates/core/src/search_prettifier/entity.rs@@ -16,7 +16,7 @@use chrono::NaiveDate;use itertools::Itertools;-use serde::{Deserialize, Serialize};+use utoipa::ToSchema;use crate::entity_index::{@@ -24,7 +24,9 @@ use crate::entity_index::{EntityMatch,};-#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct DisplayedEntity {pub title: String,diff --git a/crates/core/src/search_prettifier/mod.rs b/crates/core/src/search_prettifier/mod.rsindex 288e9894..d9a18d85 100644--- a/crates/core/src/search_prettifier/mod.rs+++ b/crates/core/src/search_prettifier/mod.rs@@ -20,7 +20,7 @@ mod stack_overflow;use std::collections::HashMap;use chrono::{NaiveDateTime, Utc};-use serde::{Deserialize, Serialize};+use url::Url;use utoipa::ToSchema;@@ -38,14 +38,18 @@ pub use entity::DisplayedEntity;pub use self::stack_overflow::{stackoverflow_snippet, StackOverflowAnswer, StackOverflowQuestion};-#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct Snippet {pub date: Option<String>,pub text: TextSnippet,}-#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, ToSchema,+)]#[serde(tag = "type", rename_all = "camelCase")]pub enum RichSnippet {StackOverflowQA {@@ -54,7 +58,9 @@ pub enum RichSnippet {},}-#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct HighlightedSpellCorrection {pub raw: String,@@ -169,7 +175,9 @@ fn generate_rich_snippet(webpage: &RetrievedWebpage) -> Option<RichSnippet> {None}-#[derive(Debug, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct DisplayedWebpage {pub title: String,@@ -185,7 +193,9 @@ pub struct DisplayedWebpage {pub likely_has_paywall: bool,}-#[derive(Debug, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct DisplayedAnswer {pub title: String,@@ -220,7 +230,9 @@ impl From<RetrievedWebpage> for DisplayedWebpage {}}-#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, ToSchema,+)]#[serde(tag = "type", content = "value", rename_all = "camelCase")]pub enum DisplayedSidebar {Entity(DisplayedEntity),diff --git a/crates/core/src/search_prettifier/stack_overflow.rs b/crates/core/src/search_prettifier/stack_overflow.rsindex c16c0b78..7475fbf8 100644--- a/crates/core/src/search_prettifier/stack_overflow.rs+++ b/crates/core/src/search_prettifier/stack_overflow.rs@@ -14,7 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use serde::{Deserialize, Serialize};use url::Url;use utoipa::ToSchema;@@ -27,7 +26,9 @@ use crate::{use super::DisplayedSidebar;use crate::Result;-#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct StackOverflowAnswer {pub body: Vec<CodeOrText>,@@ -37,13 +38,17 @@ pub struct StackOverflowAnswer {pub accepted: bool,}-#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct StackOverflowQuestion {pub body: Vec<CodeOrText>,}-#[derive(Debug, Serialize, Deserialize, Clone, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone, ToSchema,+)]#[serde(tag = "type", content = "value", rename_all = "camelCase")]pub enum CodeOrText {Code(String),diff --git a/crates/core/src/searcher/distributed.rs b/crates/core/src/searcher/distributed.rsindex 103f0868..1858e2f4 100644--- a/crates/core/src/searcher/distributed.rs+++ b/crates/core/src/searcher/distributed.rs@@ -245,7 +245,11 @@ impl SearchClient for DistributedSearcher {.await;match res {- Ok(v) => v.into_iter().flat_map(|(_, v)| v).flatten().collect(),+ Ok(v) => v+ .into_iter()+ .flat_map(|(_, v)| v.into_iter().map(|crate::bincode_utils::SerdeCompat(v)| v))+ .flatten()+ .collect(),_ => HashMap::new(),}}diff --git a/crates/core/src/searcher/mod.rs b/crates/core/src/searcher/mod.rsindex f5a49fef..60f34470 100644--- a/crates/core/src/searcher/mod.rs+++ b/crates/core/src/searcher/mod.rs@@ -22,7 +22,7 @@ pub mod local;pub use distributed::*;pub use local::*;use optics::{HostRankings, Optic};-use serde::{Deserialize, Serialize};+use utoipa::ToSchema;use crate::{@@ -32,13 +32,15 @@ use crate::{pub const NUM_RESULTS_PER_PAGE: usize = 20;-#[derive(Debug, Serialize, Deserialize)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub enum SearchResult {Websites(WebsitesResult),Bang(Box<BangHit>),}-#[derive(Debug, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct WebsitesResult {pub webpages: Vec<DisplayedWebpage>,@@ -47,7 +49,7 @@ pub struct WebsitesResult {pub has_more_results: bool,}-#[derive(Debug, Serialize, Deserialize, Clone)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Clone)]pub struct SearchQuery {pub query: String,pub page: usize,@@ -60,7 +62,7 @@ pub struct SearchQuery {pub count_results: bool,}-#[derive(Debug, Clone, Serialize, Deserialize)]+#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]pub struct InitialWebsiteResult {pub num_websites: Option<usize>,pub websites: Vec<RecallRankingWebpage>,diff --git a/crates/core/src/similar_hosts.rs b/crates/core/src/similar_hosts.rsindex a5d46ad0..c8760d3c 100644--- a/crates/core/src/similar_hosts.rs+++ b/crates/core/src/similar_hosts.rs@@ -26,7 +26,7 @@ use crate::{webpage::url_ext::UrlExt,};-#[derive(serde::Serialize, serde::Deserialize, Debug)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Debug)]pub struct ScoredNode {pub node: Node,pub score: f64,diff --git a/crates/core/src/snippet.rs b/crates/core/src/snippet.rsindex 44f311c7..aed2b2ab 100644--- a/crates/core/src/snippet.rs+++ b/crates/core/src/snippet.rs@@ -47,7 +47,17 @@ struct PassageCandidate {doc_terms: HashMap<String, u64>,}-#[derive(Default, Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, ToSchema)]+#[derive(+ Default,+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct TextSnippet {pub fragments: Vec<HighlightedFragment>,diff --git a/crates/core/src/web_spell/error_model.rs b/crates/core/src/web_spell/error_model.rsindex ad869ddc..dc8e1834 100644--- a/crates/core/src/web_spell/error_model.rs+++ b/crates/core/src/web_spell/error_model.rs@@ -14,8 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use serde::{Deserialize, Serialize};-use super::Result;use std::{collections::HashMap,@@ -24,7 +22,18 @@ use std::{path::Path,};-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]+#[derive(+ Debug,+ PartialEq,+ Eq,+ PartialOrd,+ Ord,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub enum ErrorType {Insertion(char),Deletion(char),@@ -32,7 +41,18 @@ pub enum ErrorType {Transposition(char, char),}-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]+#[derive(+ Debug,+ PartialEq,+ Eq,+ PartialOrd,+ Ord,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct ErrorSequence(Vec<ErrorType>);pub fn possible_errors(a: &str, b: &str) -> Option<ErrorSequence> {@@ -109,7 +129,7 @@ pub fn possible_errors(a: &str, b: &str) -> Option<ErrorSequence> {}}-#[derive(Debug, Serialize, Deserialize)]+#[derive(Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]struct StoredErrorModel {errors: HashMap<String, u64>,total: u64,diff --git a/crates/core/src/web_spell/mod.rs b/crates/core/src/web_spell/mod.rsindex fc26c9f2..ab8f7ceb 100644--- a/crates/core/src/web_spell/mod.rs+++ b/crates/core/src/web_spell/mod.rs@@ -37,7 +37,6 @@ use std::ops::Range;use crate::ceil_char_boundary;use crate::floor_char_boundary;use itertools::intersperse;-use serde::{Deserialize, Serialize};#[derive(Debug, thiserror::Error)]pub enum Error {@@ -50,8 +49,11 @@ pub enum Error {#[error("Serde error: {0}")]Serde(#[from] serde_json::Error),- #[error("Bincode error: {0}")]- Bincode(#[from] bincode::Error),+ #[error("Encode error: {0}")]+ Encode(#[from] bincode::error::EncodeError),++ #[error("Decode error: {0}")]+ Decode(#[from] bincode::error::DecodeError),#[error("Checker not found")]CheckerNotFound,@@ -59,13 +61,31 @@ pub enum Error {pub type Result<T, E = Error> = std::result::Result<T, E>;-#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]+#[derive(+ PartialEq,+ Eq,+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Clone,+)]pub struct Correction {original: String,pub terms: Vec<CorrectionTerm>,}-#[derive(PartialEq, Eq, Debug, Serialize, Deserialize, Clone)]+#[derive(+ PartialEq,+ Eq,+ Debug,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Clone,+)]pub enum CorrectionTerm {Corrected { orig: String, correction: String },NotCorrected(String),diff --git a/crates/core/src/web_spell/stupid_backoff.rs b/crates/core/src/web_spell/stupid_backoff.rsindex 29eb64a3..23a4d6f9 100644--- a/crates/core/src/web_spell/stupid_backoff.rs+++ b/crates/core/src/web_spell/stupid_backoff.rs@@ -23,11 +23,21 @@ use std::{};use fst::{Automaton, IntoStreamer, Streamer};-use serde::{Deserialize, Serialize};const DISCOUNT: f64 = 0.4;-#[derive(Debug, PartialEq, Eq, PartialOrd, Ord, Clone, Serialize, Deserialize)]+#[derive(+ Debug,+ PartialEq,+ Eq,+ PartialOrd,+ Ord,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Ngram {terms: Vec<String>,}@@ -129,10 +139,10 @@ impl StupidBackoffTrainer {builder.finish()?;- bincode::serialize_into(- File::create(path.as_ref().join("n_counts.bin"))?,- &self.n_counts,- )?;+ let f = File::create(path.as_ref().join("n_counts.bin"))?;+ let mut wrt = BufWriter::new(f);++ bincode::encode_into_std_write(&self.n_counts, &mut wrt, bincode::config::standard())?;Ok(())}@@ -213,8 +223,8 @@ impl StupidBackoff {let rotated_ngrams = fst::Map::new(mmap)?;let file = File::open(path.as_ref().join("n_counts.bin"))?;- let reader = std::io::BufReader::new(file);- let n_counts = bincode::deserialize_from(reader)?;+ let mut reader = std::io::BufReader::new(file);+ let n_counts = bincode::decode_from_std_read(&mut reader, bincode::config::standard())?;Ok(Self {ngrams,@@ -243,7 +253,8 @@ impl StupidBackoff {.write(true).open(path.as_ref().join("n_counts.bin"))?;- bincode::serialize_into(file, &n_counts)?;+ let mut wrt = BufWriter::new(file);+ bincode::encode_into_std_write(&n_counts, &mut wrt, bincode::config::standard())?;let file = OpenOptions::new().create(true)diff --git a/crates/core/src/web_spell/term_freqs.rs b/crates/core/src/web_spell/term_freqs.rsindex ce845c4e..6d112145 100644--- a/crates/core/src/web_spell/term_freqs.rs+++ b/crates/core/src/web_spell/term_freqs.rs@@ -16,7 +16,7 @@use super::{MergePointer, Result};use fst::{IntoStreamer, Streamer};-use serde::{Deserialize, Serialize};+use std::{collections::{BTreeMap, BinaryHeap},fs::{File, OpenOptions},@@ -156,8 +156,9 @@ impl StoredDict {}}-#[derive(Default, Serialize, Deserialize)]+#[derive(Default, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]struct Metadata {+ #[bincode(with_serde)]dicts: Vec<Uuid>,}diff --git a/crates/core/src/webgraph/centrality/betweenness.rs b/crates/core/src/webgraph/centrality/betweenness.rsindex 330bd563..7186eed2 100644--- a/crates/core/src/webgraph/centrality/betweenness.rs+++ b/crates/core/src/webgraph/centrality/betweenness.rs@@ -20,7 +20,6 @@use std::collections::{HashMap, VecDeque};use indicatif::{ProgressBar, ProgressStyle};-use serde::{Deserialize, Serialize};use crate::{intmap::IntMap,@@ -135,7 +134,9 @@ fn calculate(graph: &Webgraph, with_progress: bool) -> (HashMap<Node, f64>, i32))}-#[derive(Debug, Default, Serialize, Deserialize)]+#[derive(+ Debug, Default, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,+)]pub struct Betweenness {pub centrality: HashMap<Node, f64>,pub max_dist: usize,diff --git a/crates/core/src/webgraph/centrality/harmonic.rs b/crates/core/src/webgraph/centrality/harmonic.rsindex b6c43559..f585ed25 100644--- a/crates/core/src/webgraph/centrality/harmonic.rs+++ b/crates/core/src/webgraph/centrality/harmonic.rs@@ -138,13 +138,13 @@ fn update_centralities(centralities: &mut BTreeMap<NodeID, KahanSum>, counters:.new.get(node).map(|counter| counter.size())- .unwrap_or_default()+ .unwrap_or(1).checked_sub(counters.old.get(node).map(|counter| counter.size())- .unwrap_or_default(),+ .unwrap_or(1),).unwrap_or_default() as f64/ (t + 1) as f64;@@ -276,6 +276,14 @@ impl HarmonicCentrality {pub fn iter(&self) -> impl Iterator<Item = (&NodeID, f64)> {self.0.iter().map(|(node, centrality)| (node, *centrality))}++ pub fn len(&self) -> usize {+ self.0.len()+ }++ pub fn is_empty(&self) -> bool {+ self.len() == 0+ }}#[cfg(test)]diff --git a/crates/core/src/webgraph/centrality/mod.rs b/crates/core/src/webgraph/centrality/mod.rsindex 012a8847..fe84d521 100644--- a/crates/core/src/webgraph/centrality/mod.rs+++ b/crates/core/src/webgraph/centrality/mod.rs@@ -14,9 +14,15 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use crate::kv::{rocksdb_store::RocksDbStore, Kv};+use std::{cmp::Reverse, collections::BinaryHeap, fs::File, path::Path};-use super::NodeID;+use crate::{+ external_sort::ExternalSorter,+ kv::{rocksdb_store::RocksDbStore, Kv},+ SortableFloat,+};++use super::{Node, NodeID};pub mod approx_harmonic;pub mod betweenness;@@ -24,33 +30,84 @@ pub mod derived_harmonic;pub mod harmonic;#[derive(Debug, Clone, Copy)]-pub enum TopHosts {+pub enum TopNodes {Top(usize),Fraction(f64),}-pub fn top_hosts(host_centrality: &RocksDbStore<NodeID, f64>, top: TopHosts) -> Vec<NodeID> {- let mut hosts = host_centrality- .iter()- .map(|(id, centrality)| {- if !centrality.is_finite() {- (id, 0.0)- } else {- (id, centrality)- }- })- .collect::<Vec<_>>();-- hosts.sort_by(|(_, a), (_, b)| b.total_cmp(a));-+pub fn top_nodes(host_centrality: &RocksDbStore<NodeID, f64>, top: TopNodes) -> Vec<(NodeID, f64)> {let num_hosts = match top {- TopHosts::Top(abs) => abs,- TopHosts::Fraction(frac) => (hosts.len() as f64 * frac) as usize,+ TopNodes::Top(abs) => abs,+ TopNodes::Fraction(frac) => (host_centrality.approx_len() as f64 * frac) as usize,};- hosts+ let mut top: BinaryHeap<Reverse<(SortableFloat, NodeID)>> = BinaryHeap::new();++ for (id, centrality) in host_centrality.iter() {+ if top.len() >= num_hosts {+ let mut min = top.peek_mut().unwrap();++ if centrality > min.0 .0 .0 {+ min.0 .1 = id;+ min.0 .0 .0 = centrality;+ }+ } else {+ top.push(Reverse((SortableFloat(centrality), id)));+ }+ }++ top.into_sorted_vec().into_iter()- .take(num_hosts)- .map(|(id, _)| id)+ .map(|Reverse((SortableFloat(c), id))| (id, c))+ .rev().collect()}++pub fn store_csv<P: AsRef<Path>>(data: Vec<(Node, f64)>, output: P) {+ let csv_file = File::options()+ .write(true)+ .create(true)+ .truncate(true)+ .open(output)+ .unwrap();++ let mut data = data;+ data.sort_by(|(_, a), (_, b)| b.partial_cmp(a).unwrap_or(std::cmp::Ordering::Equal));+ let mut wtr = csv::Writer::from_writer(csv_file);+ for (node, centrality) in data {+ wtr.write_record(&[node.as_str().to_string(), centrality.to_string()])+ .unwrap();+ }+ wtr.flush().unwrap();+}++pub fn store_harmonic<I, P>(centralities: I, output: P) -> RocksDbStore<NodeID, f64>+where+ I: Iterator<Item = (NodeID, f64)>,+ P: AsRef<Path>,+{+ let store = RocksDbStore::open(output.as_ref().join("harmonic"));++ for (node_id, centrality) in centralities {+ store.insert(node_id, centrality);+ }+ store.flush();++ let rank_store: RocksDbStore<crate::webgraph::NodeID, u64> =+ RocksDbStore::open(output.as_ref().join("harmonic_rank"));+ for (rank, node_id) in ExternalSorter::new()+ .with_chunk_size(100_000_000)+ .sort(+ store+ .iter()+ .map(|(node_id, centrality)| (Reverse(SortableFloat(centrality)), node_id)),+ )+ .unwrap()+ .enumerate()+ .map(|(rank, (_, node_id))| (rank, node_id))+ {+ rank_store.insert(node_id, rank as u64);+ }++ store+}diff --git a/crates/core/src/webgraph/edge.rs b/crates/core/src/webgraph/edge.rsindex 0d7e7ba9..71972c75 100644--- a/crates/core/src/webgraph/edge.rs+++ b/crates/core/src/webgraph/edge.rs@@ -81,7 +81,18 @@ where}}-#[derive(Debug, Clone, serde::Serialize, serde::Deserialize, PartialEq, Eq, Hash, ToSchema)]+#[derive(+ Debug,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Hash,+ ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct FullEdge {pub from: Node,diff --git a/crates/core/src/webgraph/id_node_db.rs b/crates/core/src/webgraph/id_node_db.rsindex a6ef4f28..e92d5014 100644--- a/crates/core/src/webgraph/id_node_db.rs+++ b/crates/core/src/webgraph/id_node_db.rs@@ -19,7 +19,7 @@ use std::{collections::BTreeMap, io::Write, path::Path};use super::{Node, NodeID};use crate::Result;-#[derive(serde::Serialize, serde::Deserialize)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]struct StoredPtr {file_offset: usize,len: usize,@@ -31,7 +31,7 @@ impl StoredPtr {}}-#[derive(serde::Serialize, serde::Deserialize)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode)]struct Index {map: BTreeMap<NodeID, StoredPtr>,}@@ -46,9 +46,10 @@ impl Index {fn open_or_new<P: AsRef<Path>>(path: P) -> Result<Self> {if path.as_ref().exists() {let file = std::fs::File::open(path)?;- let reader = std::io::BufReader::new(file);+ let mut reader = std::io::BufReader::new(file);- Ok(bincode::deserialize_from(reader)?)+ let res = bincode::decode_from_std_read(&mut reader, bincode::config::standard())?;+ Ok(res)} else {Ok(Self::new())}@@ -56,9 +57,9 @@ impl Index {fn save<P: AsRef<Path>>(&self, path: P) -> Result<()> {let file = std::fs::File::create(path)?;- let writer = std::io::BufWriter::new(file);+ let mut writer = std::io::BufWriter::new(file);- bincode::serialize_into(writer, self)?;+ bincode::encode_into_std_write(self, &mut writer, bincode::config::standard())?;Ok(())}@@ -110,7 +111,7 @@ impl FileStore {}fn write(&mut self, node: &Node) -> Result<StoredPtr> {- let bytes = bincode::serialize(node)?;+ let bytes = bincode::encode_to_vec(node, bincode::config::standard())?;self.writer.write_all(&bytes)?;let file_offset = self.cur_offset;@@ -130,7 +131,7 @@ impl FileStore {}let bytes = &self.mmap[range];- let node = bincode::deserialize(bytes)?;+ let (node, _) = bincode::decode_from_slice(bytes, bincode::config::standard())?;Ok(Some(node))}diff --git a/crates/core/src/webgraph/mod.rs b/crates/core/src/webgraph/mod.rsindex 3b834956..ecb53f34 100644--- a/crates/core/src/webgraph/mod.rs+++ b/crates/core/src/webgraph/mod.rs@@ -47,7 +47,7 @@ mod writer;type SegmentID = String;-#[derive(serde::Serialize, serde::Deserialize, Default)]+#[derive(serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Default)]struct Meta {comitted_segments: Vec<SegmentID>,}@@ -94,6 +94,10 @@ pub struct Webgraph {}impl Webgraph {+ pub fn builder<P: AsRef<Path>>(path: P) -> WebgraphBuilder {+ WebgraphBuilder::new(path)+ }+fn meta<P: AsRef<Path>>(path: P) -> Meta {let meta_path = path.as_ref().join("metadata.json");Meta::open(meta_path)@@ -305,7 +309,7 @@ impl Webgraph {}#[cfg(test)]-mod test {+pub mod tests {use super::*;fn test_edges() -> Vec<(Node, Node, String)> {@@ -318,12 +322,12 @@ mod test {]}- fn test_graph() -> Webgraph {- // ┌────┐- // │ │- // ┌───A◄─┐ │- // │ │ │- // ▼ │ │+ pub fn test_graph() -> Webgraph {+ // ┌------┐+ // │ │+ // ┌───A◄─┐ │+ // │ │ │+ // ▼ │ │// B─────►C◄┘// ▲// │diff --git a/crates/core/src/webgraph/node.rs b/crates/core/src/webgraph/node.rsindex fb3f91da..0bb927d0 100644--- a/crates/core/src/webgraph/node.rs+++ b/crates/core/src/webgraph/node.rs@@ -20,7 +20,18 @@ use utoipa::ToSchema;use crate::{intmap, webpage::url_ext::UrlExt};#[derive(- Debug, Clone, Copy, serde::Serialize, serde::Deserialize, PartialEq, Eq, PartialOrd, Ord, Hash,+ Debug,+ Clone,+ Copy,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ PartialOrd,+ Ord,+ Hash,)]pub struct NodeID(u64);@@ -42,7 +53,17 @@ impl From<u64> for NodeID {}}-#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct FullNodeID {pub prefix: NodeID,pub id: NodeID,@@ -79,6 +100,8 @@ impl intmap::Key for NodeID {Hash,serde::Serialize,serde::Deserialize,+ bincode::Encode,+ bincode::Decode,ToSchema,)]#[serde(rename_all = "camelCase")]diff --git a/crates/core/src/webgraph/store.rs b/crates/core/src/webgraph/store.rsindex f9ad6ea8..9f311944 100644--- a/crates/core/src/webgraph/store.rs+++ b/crates/core/src/webgraph/store.rs@@ -22,7 +22,17 @@ use rocksdb::BlockBasedOptions;use super::{Compression, Edge, EdgeLabel, FullNodeID, InnerEdge, NodeID};-#[derive(Debug, Clone, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)]+#[derive(+ Debug,+ Clone,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]struct SerializedEdge {from_prefix: NodeID,to_prefix: NodeID,@@ -87,11 +97,14 @@ impl EdgeStoreWriter {for edge in edges {let value_bytes = L::to_bytes(&edge.label).unwrap();- let value_bytes = bincode::serialize(&SerializedEdge {- from_prefix: edge.from.prefix,- to_prefix: edge.to.prefix,- label: value_bytes.clone(),- })+ let value_bytes = bincode::encode_to_vec(+ &SerializedEdge {+ from_prefix: edge.from.prefix,+ to_prefix: edge.to.prefix,+ label: value_bytes.clone(),+ },+ bincode::config::standard(),+ ).unwrap();let key_bytes = if self.reversed {@@ -141,7 +154,8 @@ impl EdgeStoreWriter {)};- let val: SerializedEdge = bincode::deserialize(&val).unwrap();+ let (val, _): (SerializedEdge, _) =+ bincode::decode_from_slice(&val, bincode::config::standard()).unwrap();Some(InnerEdge {from: FullNodeID {@@ -413,8 +427,10 @@ impl EdgeStore {});}- let edge_labels_bytes = bincode::serialize(&edge_labels).unwrap();- let edge_nodes_bytes = bincode::serialize(&edge_nodes).unwrap();+ let edge_labels_bytes =+ bincode::encode_to_vec(&edge_labels, bincode::config::standard()).unwrap();+ let edge_nodes_bytes =+ bincode::encode_to_vec(&edge_nodes, bincode::config::standard()).unwrap();let edge_labels_bytes = self.compression.compress(&edge_labels_bytes);let edge_nodes_bytes = self.compression.compress(&edge_nodes_bytes);@@ -435,7 +451,7 @@ impl EdgeStore {.put_cf_opt(node_cf,node_bytes,- bincode::serialize(&node_range).unwrap(),+ bincode::encode_to_vec(node_range, bincode::config::standard()).unwrap(),&opt,).unwrap();@@ -444,7 +460,7 @@ impl EdgeStore {.put_cf_opt(label_cf,node_bytes,- bincode::serialize(&label_range).unwrap(),+ bincode::encode_to_vec(label_range, bincode::config::standard()).unwrap(),&opt,).unwrap();@@ -528,16 +544,26 @@ impl EdgeStore {self.ranges.get_cf_opt(edge_cf, node_bytes, &opts).unwrap(),) {(Some(node_range_bytes), Some(edge_range_bytes)) => {- let node_range = bincode::deserialize::<Range<usize>>(&node_range_bytes).unwrap();- let edge_range = bincode::deserialize::<Range<usize>>(&edge_range_bytes).unwrap();+ let (node_range, _) = bincode::decode_from_slice::<Range<usize>, _>(+ &node_range_bytes,+ bincode::config::standard(),+ )+ .unwrap();+ let (edge_range, _) = bincode::decode_from_slice::<Range<usize>, _>(+ &edge_range_bytes,+ bincode::config::standard(),+ )+ .unwrap();let edge_labels = &self.edge_labels[edge_range];let edge_labels = self.compression.decompress(edge_labels);- let edge_labels: Vec<_> = bincode::deserialize(&edge_labels).unwrap();+ let (edge_labels, _): (Vec<_>, _) =+ bincode::decode_from_slice(&edge_labels, bincode::config::standard()).unwrap();let edge_nodes = &self.edge_nodes[node_range];let edge_nodes = self.compression.decompress(edge_nodes);- let edge_nodes: Vec<_> = bincode::deserialize(&edge_nodes).unwrap();+ let (edge_nodes, _): (Vec<_>, _) =+ bincode::decode_from_slice(&edge_nodes, bincode::config::standard()).unwrap();edge_labels.into_iter()@@ -573,11 +599,16 @@ impl EdgeStore {match self.ranges.get_cf_opt(node_cf, node_bytes, &opts).unwrap() {Some(node_range_bytes) => {- let node_range = bincode::deserialize::<Range<usize>>(&node_range_bytes).unwrap();+ let (node_range, _) = bincode::decode_from_slice::<Range<usize>, _>(+ &node_range_bytes,+ bincode::config::standard(),+ )+ .unwrap();let edge_nodes = &self.edge_nodes[node_range];let edge_nodes = self.compression.decompress(edge_nodes);- let edge_nodes: Vec<_> = bincode::deserialize(&edge_nodes).unwrap();+ let (edge_nodes, _): (Vec<_>, _) =+ bincode::decode_from_slice(&edge_nodes, bincode::config::standard()).unwrap();edge_nodes.into_iter()@@ -621,10 +652,15 @@ impl EdgeStore {let node = u64::from_le_bytes((*key).try_into().unwrap());let node = NodeID::from(node);- let node_range = bincode::deserialize::<Range<usize>>(&val).unwrap();+ let (node_range, _) = bincode::decode_from_slice::<Range<usize>, _>(+ &val,+ bincode::config::standard(),+ )+ .unwrap();let edge_nodes = &self.edge_nodes[node_range];let edge_nodes = self.compression.decompress(edge_nodes);- let edge_nodes: Vec<_> = bincode::deserialize(&edge_nodes).unwrap();+ let (edge_nodes, _): (Vec<_>, _) =+ bincode::decode_from_slice(&edge_nodes, bincode::config::standard()).unwrap();edge_nodes.into_iter().map(move |other| {if self.reversed {diff --git a/crates/core/src/webpage/html/fn_cache.rs b/crates/core/src/webpage/html/fn_cache.rsindex fee3abae..daf66c59 100644--- a/crates/core/src/webpage/html/fn_cache.rs+++ b/crates/core/src/webpage/html/fn_cache.rs@@ -19,7 +19,6 @@ use tantivy::tokenizer::PreTokenizedString;use super::{find_recipe_first_ingredient_tag_id, Html};-#[macro_export]macro_rules! cache {($($fn:ident -> $res:ty),*$(,)?) => {/// Dynamically compute the different webpage functionsdiff --git a/crates/core/src/webpage/region.rs b/crates/core/src/webpage/region.rsindex f7496226..30722b3f 100644--- a/crates/core/src/webpage/region.rs+++ b/crates/core/src/webpage/region.rs@@ -16,14 +16,26 @@use std::{collections::HashMap, fs::File, io::Write, path::Path};-use serde::{Deserialize, Serialize};use utoipa::ToSchema;use crate::{Error, Result};use super::Webpage;-#[derive(Debug, PartialEq, Eq, Clone, Copy, Hash, Serialize, Deserialize, Default, ToSchema)]+#[derive(+ Debug,+ PartialEq,+ Eq,+ Clone,+ Copy,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Default,+ ToSchema,+)]pub enum Region {#[default]All,@@ -123,7 +135,9 @@ impl Region {}}-#[derive(Serialize, Deserialize, Default, Clone)]+#[derive(+ serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, Default, Clone,+)]pub struct RegionCount {map: HashMap<Region, u64>,fast_count: Vec<Option<u64>>,diff --git a/crates/core/src/webpage/safety_classifier.rs b/crates/core/src/webpage/safety_classifier.rsindex 2597301e..8c230025 100644--- a/crates/core/src/webpage/safety_classifier.rs+++ b/crates/core/src/webpage/safety_classifier.rs@@ -26,7 +26,18 @@ use crate::Result;const MAX_NUM_WORDS: usize = 100;#[derive(- Debug, Clone, Copy, PartialEq, Eq, Hash, PartialOrd, Ord, serde::Serialize, serde::Deserialize,+ Debug,+ Clone,+ Copy,+ PartialEq,+ Eq,+ Hash,+ PartialOrd,+ Ord,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,)]pub enum Label {SFW,@@ -57,7 +68,7 @@ impl TryFrom<&str> for Label {impl naive_bayes::Label for Label {}-#[derive(Debug, serde::Deserialize, serde::Serialize)]+#[derive(Debug, bincode::Encode, bincode::Decode, serde::Serialize, serde::Deserialize)]pub struct Datapoint {pub label: Label,pub text: String,@@ -93,7 +104,7 @@ pub struct Evaluation {pub f1: f64,}-#[derive(serde::Deserialize, serde::Serialize)]+#[derive(bincode::Encode, bincode::Decode)]pub struct Model {pipeline: naive_bayes::Pipeline<Label>,}@@ -169,22 +180,22 @@ impl Model {}pub fn save<P: AsRef<Path>>(self, path: P) -> Result<()> {- let file = OpenOptions::new()+ let mut file = OpenOptions::new().write(true).create(true).truncate(true).open(path)?;- bincode::serialize_into(file, &self)?;+ bincode::encode_into_std_write(&self, &mut file, bincode::config::standard())?;Ok(())}pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {let file = OpenOptions::new().read(true).open(path)?;- let reader = std::io::BufReader::new(file);+ let mut reader = std::io::BufReader::new(file);- let model = bincode::deserialize_from(reader)?;+ let model = bincode::decode_from_std_read(&mut reader, bincode::config::standard())?;Ok(model)}diff --git a/crates/core/src/webpage/schema_org/microdata.rs b/crates/core/src/webpage/schema_org/microdata.rsindex aa8383a2..1f54f4af 100644--- a/crates/core/src/webpage/schema_org/microdata.rs+++ b/crates/core/src/webpage/schema_org/microdata.rs@@ -29,8 +29,11 @@ pub enum Error {#[error("Html node was expected to have an itemscope attribute, but did not have one.")]ExpectedItemScope,- #[error("Error while serializing/deserializing to/from bytes")]- Serialization(#[from] bincode::Error),+ #[error("Error while encoding to bytes: {0}")]+ Encode(#[from] bincode::error::EncodeError),++ #[error("Error while decoding from bytes: {0}")]+ Decode(#[from] bincode::error::DecodeError),#[error("Could not convert to/from JSON")]Json(#[from] serde_json::Error),diff --git a/crates/core/src/webpage/schema_org/mod.rs b/crates/core/src/webpage/schema_org/mod.rsindex c33f53bb..08252376 100644--- a/crates/core/src/webpage/schema_org/mod.rs+++ b/crates/core/src/webpage/schema_org/mod.rs@@ -17,7 +17,6 @@use std::collections::HashMap;use kuchiki::NodeRef;-use serde::{Deserialize, Serialize};use crate::tokenizer::FlattenedJson;use crate::Result;@@ -25,7 +24,16 @@ use crate::Result;mod json_ld;mod microdata;-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[derive(+ Debug,+ Clone,+ PartialEq,+ Eq,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub enum Property {String(String),Item(Item),@@ -46,7 +54,16 @@ impl Property {}}-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[derive(+ Debug,+ Clone,+ PartialEq,+ Eq,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]#[serde(untagged)]enum SingleMap {Leaf(String),@@ -85,7 +102,7 @@ impl From<Property> for SingleMap {}}-#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]+#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)]pub struct RawItem {#[serde(rename = "@type")]itemtype: Option<RawOneOrMany<String>>,@@ -93,7 +110,7 @@ pub struct RawItem {properties: HashMap<String, RawOneOrMany<RawProperty>>,}-#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]+#[derive(Debug, Clone, PartialEq, Eq, serde::Deserialize)]#[serde(untagged)]enum RawProperty {String(String),@@ -126,7 +143,16 @@ impl From<RawProperty> for Property {}}-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[derive(+ Debug,+ Clone,+ PartialEq,+ Eq,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Item {pub itemtype: Option<OneOrMany<String>>,pub properties: HashMap<String, OneOrMany<Property>>,@@ -170,8 +196,18 @@ impl From<RawItem> for Item {}}-#[derive(Serialize, Deserialize, PartialEq, Eq, Debug, Clone, Hash)]-pub enum OneOrMany<T> {+#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Debug,+ Clone,+ Hash,+)]+pub enum OneOrMany<T: 'static> {One(T),Many(Vec<T>),}@@ -192,9 +228,19 @@ impl<T> OneOrMany<T> {}}-#[derive(Deserialize, Serialize, PartialEq, Eq, Debug, Clone, Hash)]+#[derive(+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ PartialEq,+ Eq,+ Debug,+ Clone,+ Hash,+)]#[serde(untagged)]-enum RawOneOrMany<T> {+enum RawOneOrMany<T: 'static> {One(T),Many(Vec<T>),}diff --git a/crates/core/src/widgets/calculator.rs b/crates/core/src/widgets/calculator.rsindex 3195fa24..66217236 100644--- a/crates/core/src/widgets/calculator.rs+++ b/crates/core/src/widgets/calculator.rs@@ -18,7 +18,7 @@ use crate::widgets::Error;use anyhow::{anyhow, Result};use hashbrown::HashMap;use rand::Rng;-use serde::{Deserialize, Serialize};+use std::{fmt::Debug,sync::{atomic::AtomicUsize, Arc, Mutex},@@ -28,7 +28,9 @@ use utoipa::ToSchema;static DICE_REGEX: once_cell::sync::Lazy<regex::Regex> =once_cell::sync::Lazy::new(|| regex::Regex::new(r"^d[0-9]+").unwrap());-#[derive(Debug, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct Calculation {pub input: String,diff --git a/crates/core/src/widgets/mod.rs b/crates/core/src/widgets/mod.rsindex be9470bf..337319c3 100644--- a/crates/core/src/widgets/mod.rs+++ b/crates/core/src/widgets/mod.rs@@ -14,7 +14,6 @@// You should have received a copy of the GNU Affero General Public License// along with this program. If not, see <https://www.gnu.org/licenses/>.-use serde::{Deserialize, Serialize};use thiserror::Error;use utoipa::ToSchema;@@ -81,7 +80,9 @@ impl Widgets {}}-#[derive(Debug, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]#[serde(tag = "type", content = "value", rename_all = "camelCase")]pub enum Widget {Calculator(Calculation),diff --git a/crates/core/src/widgets/thesaurus.rs b/crates/core/src/widgets/thesaurus.rsindex 88737db4..68d23cc7 100644--- a/crates/core/src/widgets/thesaurus.rs+++ b/crates/core/src/widgets/thesaurus.rs@@ -16,7 +16,7 @@use itertools::Itertools;use regex::Regex;-use serde::{Deserialize, Serialize};+use std::{fs::File,io::{BufRead, BufReader},@@ -33,7 +33,18 @@ static VALUE_STR_REGEX: once_cell::sync::Lazy<Regex> = once_cell::sync::Lazy::neregex::Regex::new(r#""(.*)"@"#).expect("Failed to compile regex")});-#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug,+ Clone,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ ToSchema,+)]#[serde(rename_all = "camelCase", transparent)]pub struct Lemma(String);@@ -49,15 +60,48 @@ impl Lemma {#[derive(Debug, Clone, PartialEq, Eq, Hash)]struct Id(String);-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug,+ Clone,+ PartialEq,+ Eq,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ ToSchema,+)]#[serde(rename_all = "camelCase", transparent)]pub struct Definition(String);-#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug,+ Clone,+ PartialEq,+ Eq,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ ToSchema,+)]#[serde(rename_all = "camelCase", transparent)]pub struct Example(String);-#[derive(Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug,+ Clone,+ PartialOrd,+ Ord,+ PartialEq,+ Eq,+ Hash,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ ToSchema,+)]#[serde(rename_all = "camelCase")]pub enum PartOfSpeech {Noun,@@ -502,7 +546,9 @@ impl Dictionary {}}-#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct WordMeaning {pub definition: Definition,@@ -510,7 +556,9 @@ pub struct WordMeaning {pub similar: Vec<Lemma>,}-#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct PartOfSpeechMeaning {pub pos: PartOfSpeech,@@ -537,7 +585,9 @@ impl PartialOrd for PartOfSpeechMeaning {}}-#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]+#[derive(+ Debug, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode, ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct ThesaurusWidget {pub term: Lemma,diff --git a/crates/optics-lsp/Cargo.lock b/crates/optics-lsp/Cargo.lockindex cf57baaa..c7bb15ab 100644--- a/crates/optics-lsp/Cargo.lock+++ b/crates/optics-lsp/Cargo.lock@@ -32,6 +32,24 @@ version = "0.5.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3a8241f3ebb85c056b509d4327ad0358fbbba6ffb340bf388f26350aeda225b1"+[[package]]+name = "bincode"+version = "2.0.0-rc.3"+source = "git+https://github.com/bincode-org/bincode?rev=8f65c60#8f65c60f192ff6093e6a1326b9d078a3b26c9a29"+dependencies = [+ "bincode_derive",+ "serde",+ "unty",+]++[[package]]+name = "bincode_derive"+version = "2.0.0-rc.3"+source = "git+https://github.com/bincode-org/bincode?rev=8f65c60#8f65c60f192ff6093e6a1326b9d078a3b26c9a29"+dependencies = [+ "virtue",+]+[[package]]name = "bit-set"version = "0.5.3"@@ -409,6 +427,7 @@ checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"name = "optics"version = "0.1.0"dependencies = [+ "bincode","itertools 0.11.0","lalrpop","lalrpop-util",@@ -792,6 +811,12 @@ version = "0.2.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "f962df74c8c05a667b5ee8bcf162993134c104e96440b663c8daa176dc772d8c"+[[package]]+name = "unty"+version = "0.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a1a88342087869553c259588a3ec9ca73ce9b2d538b7051ba5789ff236b6c129"+[[package]]name = "url"version = "2.3.1"@@ -835,6 +860,12 @@ version = "0.9.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"+[[package]]+name = "virtue"+version = "0.0.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3b6826a786a78cf1bb0937507b5551fb6f827d66269a24b00af0de247b19bbc7"+[[package]]name = "wasi"version = "0.11.0+wasi-snapshot-preview1"diff --git a/crates/optics/Cargo.toml b/crates/optics/Cargo.tomlindex f0c05efe..3d050d35 100644--- a/crates/optics/Cargo.toml+++ b/crates/optics/Cargo.toml@@ -12,6 +12,7 @@ itertools = { workspace = true }logos = { workspace = true }serde = { workspace = true }utoipa = { workspace = true }+bincode = { workspace = true }[build-dependencies]lalrpop = { workspace = true }diff --git a/crates/optics/src/ast.rs b/crates/optics/src/ast.rsindex bf6f827b..507b4acb 100644--- a/crates/optics/src/ast.rs+++ b/crates/optics/src/ast.rs@@ -17,8 +17,6 @@use super::Error;use super::Result as ModResult;use lalrpop_util::lalrpop_mod;-use serde::Deserialize;-use serde::Serialize;use std::fmt::Display;use super::lexer;@@ -28,12 +26,16 @@ lalrpop_mod!(pub parser, "/parser.rs");pub static PARSER: once_cell::sync::Lazy<parser::BlocksParser> =once_cell::sync::Lazy::new(parser::BlocksParser::new);-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]+#[derive(+ Debug, PartialEq, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,+)]pub enum RankingTarget {Signal(String),}-#[derive(Debug, PartialEq, Clone, Serialize, Deserialize)]+#[derive(+ Debug, PartialEq, Clone, serde::Serialize, serde::Deserialize, bincode::Encode, bincode::Decode,+)]pub struct RankingCoeff {pub target: RankingTarget,pub value: f64,diff --git a/crates/optics/src/lib.rs b/crates/optics/src/lib.rsindex 331d4b34..923837ff 100644--- a/crates/optics/src/lib.rs+++ b/crates/optics/src/lib.rs@@ -19,7 +19,6 @@ mod lexer;use ast::RankingCoeff;use itertools::Itertools;-use serde::{Deserialize, Serialize};use std::fmt::Display;use thiserror::Error;use utoipa::ToSchema;@@ -145,7 +144,16 @@ impl From<RawAction> for Action {}}-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]+#[derive(+ Debug,+ PartialEq,+ Eq,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Matching {pub pattern: Vec<PatternPart>,pub location: MatchLocation,@@ -215,7 +223,16 @@ impl TryFrom<RawMatchPart> for Matching {}}-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]+#[derive(+ Debug,+ PartialEq,+ Eq,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]enum PatternToken {Raw(String),@@ -268,7 +285,16 @@ impl PatternToken {}}-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]+#[derive(+ Debug,+ PartialEq,+ Eq,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub enum PatternPart {Raw(String),Wildcard,@@ -285,7 +311,16 @@ impl Display for PatternPart {}}-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]+#[derive(+ Debug,+ PartialEq,+ Eq,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub enum MatchLocation {Site,Url,@@ -297,7 +332,17 @@ pub enum MatchLocation {Schema,}-#[derive(Debug, PartialEq, Eq, Clone, Copy, Serialize, Deserialize)]+#[derive(+ Debug,+ PartialEq,+ Eq,+ Clone,+ Copy,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub enum Action {Boost(u64),Downrank(u64),@@ -318,7 +363,16 @@ impl Display for Action {}}-#[derive(Debug, PartialEq, Default, Clone, Serialize, Deserialize)]+#[derive(+ Debug,+ PartialEq,+ Default,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Optic {pub rankings: Vec<RankingCoeff>,pub host_rankings: HostRankings,@@ -350,7 +404,16 @@ impl Display for Optic {}}-#[derive(Debug, PartialEq, Eq, Clone, Serialize, Deserialize)]+#[derive(+ Debug,+ PartialEq,+ Eq,+ Clone,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+)]pub struct Rule {/// A list of matchings, structured as an OR of ANDs (i.e. the rule matches if all of the matchings inside one list match).pub matches: Vec<Vec<Matching>>,@@ -411,7 +474,17 @@ impl Display for Rule {}}-#[derive(Debug, PartialEq, Default, Serialize, Deserialize, Clone, ToSchema)]+#[derive(+ Debug,+ PartialEq,+ Default,+ serde::Serialize,+ serde::Deserialize,+ bincode::Encode,+ bincode::Decode,+ Clone,+ ToSchema,+)]#[serde(rename_all = "camelCase")]pub struct HostRankings {pub liked: Vec<String>,diff --git a/scripts/licenses/licenses.toml b/scripts/licenses/licenses.tomlindex c9ae6f1c..552bba02 100644--- a/scripts/licenses/licenses.toml+++ b/scripts/licenses/licenses.toml@@ -15,6 +15,7 @@ accepted = ["Unlicense","zlib-acknowledgement","GPL-3.0", # aglpv3 software can link glpv3 libraries under section 13 of gplv3+ "BSL-1.0",]workarounds = ["ring"]@@ -22,5 +23,8 @@ workarounds = ["ring"][stract]accepted = ["AGPL-3.0"]+[client-wasm]+accepted = ["AGPL-3.0"]+[wav]accepted = ["LGPL-3.0"]