web search done right
dependency cleanup
Mikkel Denker5c35f5d10ccc
diff --git a/Cargo.lock b/Cargo.lockindex 7655040e..6a1925ef 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -20,7 +20,7 @@ version = "0.7.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"dependencies = [- "getrandom",+ "getrandom 0.2.6","once_cell","version_check",]@@ -49,15 +49,6 @@ version = "1.0.58"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704"-[[package]]-name = "ascii-canvas"-version = "3.0.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6"-dependencies = [- "term",-]-[[package]]name = "askama"version = "0.11.1"@@ -237,21 +228,6 @@ dependencies = ["shlex",]-[[package]]-name = "bit-set"-version = "0.5.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"-dependencies = [- "bit-vec",-]--[[package]]-name = "bit-vec"-version = "0.6.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"-[[package]]name = "bit_field"version = "0.10.1"@@ -273,15 +249,6 @@ dependencies = ["crunchy",]-[[package]]-name = "block-buffer"-version = "0.9.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"-dependencies = [- "generic-array",-]-[[package]]name = "bstr"version = "0.2.17"@@ -378,7 +345,6 @@ dependencies = ["js-sys","num-integer","num-traits",- "serde","time 0.1.44","wasm-bindgen","winapi",@@ -462,6 +428,12 @@ dependencies = ["winapi",]+[[package]]+name = "convert_case"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"+[[package]]name = "core-foundation"version = "0.9.3"@@ -478,15 +450,6 @@ version = "0.8.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"-[[package]]-name = "cpufeatures"-version = "0.2.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"-dependencies = [- "libc",-]-[[package]]name = "crc32fast"version = "1.3.2"@@ -548,13 +511,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"[[package]]-name = "crypto-mac"-version = "0.11.1"+name = "cssparser"+version = "0.27.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"+checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"dependencies = [- "generic-array",- "subtle",+ "cssparser-macros",+ "dtoa-short",+ "itoa 0.4.8",+ "matches",+ "phf",+ "proc-macro2",+ "quote",+ "smallvec",+ "syn",+]++[[package]]+name = "cssparser-macros"+version = "0.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e"+dependencies = [+ "quote",+ "syn",][[package]]@@ -611,9 +591,7 @@ dependencies = ["image","indicatif","itertools",- "lalrpop",- "lalrpop-util",- "lazy_static",+ "kuchiki","logos","lru","maplit",@@ -626,11 +604,8 @@ dependencies = ["regex","reqwest","rocksdb",- "rusoto_core",- "rusoto_s3","serde","serde_json",- "sled","tantivy","thiserror","tokio",@@ -650,6 +625,19 @@ dependencies = ["adler32",]+[[package]]+name = "derive_more"+version = "0.99.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321"+dependencies = [+ "convert_case",+ "proc-macro2",+ "quote",+ "rustc_version",+ "syn",+]+[[package]]name = "diff"version = "0.1.12"@@ -657,56 +645,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499"[[package]]-name = "digest"-version = "0.9.0"+name = "downcast-rs"+version = "1.2.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"-dependencies = [- "generic-array",-]+checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"[[package]]-name = "dirs-next"-version = "2.0.0"+name = "dtoa"+version = "0.4.8"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"-dependencies = [- "cfg-if",- "dirs-sys-next",-]+checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"[[package]]-name = "dirs-sys-next"-version = "0.1.2"+name = "dtoa-short"+version = "0.3.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"+checksum = "bde03329ae10e79ede66c9ce4dc930aa8599043b0743008548680f25b91502d6"dependencies = [- "libc",- "redox_users",- "winapi",+ "dtoa",]-[[package]]-name = "downcast-rs"-version = "1.2.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"-[[package]]name = "either"version = "1.6.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"-[[package]]-name = "ena"-version = "0.14.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3"-dependencies = [- "log",-]-[[package]]name = "encode_unicode"version = "0.3.6"@@ -746,7 +710,7 @@ checksum = "ec3245a0ca564e7f3c797d20d833a6870f57a728ac967d5225b3ffdef4465011"dependencies = ["lazy_static","log",- "rand",+ "rand 0.8.5",][[package]]@@ -774,12 +738,6 @@ dependencies = ["instant",]-[[package]]-name = "fixedbitset"-version = "0.4.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"-[[package]]name = "flate2"version = "1.0.24"@@ -853,6 +811,16 @@ dependencies = ["utf8-ranges",]+[[package]]+name = "futf"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"+dependencies = [+ "mac",+ "new_debug_unreachable",+]+[[package]]name = "futures"version = "0.3.21"@@ -965,13 +933,14 @@ dependencies = [][[package]]-name = "generic-array"-version = "0.14.6"+name = "getrandom"+version = "0.1.16"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"dependencies = [- "typenum",- "version_check",+ "cfg-if",+ "libc",+ "wasi 0.9.0+wasi-snapshot-preview1",][[package]]@@ -1062,28 +1031,26 @@ dependencies = [][[package]]-name = "hex"-version = "0.4.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"--[[package]]-name = "hmac"-version = "0.11.0"+name = "html-escape"+version = "0.2.11"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"+checksum = "b8e7479fa1ef38eb49fb6a42c426be515df2d063f06cb8efd3e50af073dbc26c"dependencies = [- "crypto-mac",- "digest",+ "utf8-width",][[package]]-name = "html-escape"-version = "0.2.11"+name = "html5ever"+version = "0.25.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b8e7479fa1ef38eb49fb6a42c426be515df2d063f06cb8efd3e50af073dbc26c"+checksum = "e5c13fb08e5d4dfc151ee5e88bae63f7773d61852f3bdc73c9f4b9e1bde03148"dependencies = [- "utf8-width",+ "log",+ "mac",+ "markup5ever",+ "proc-macro2",+ "quote",+ "syn",][[package]]@@ -1304,35 +1271,15 @@ dependencies = [][[package]]-name = "lalrpop"-version = "0.19.8"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b30455341b0e18f276fa64540aff54deafb54c589de6aca68659c63dd2d5d823"-dependencies = [- "ascii-canvas",- "atty",- "bit-set",- "diff",- "ena",- "itertools",- "lalrpop-util",- "petgraph",- "pico-args",- "regex",- "regex-syntax 0.6.27",- "string_cache",- "term",- "tiny-keccak",- "unicode-xid",-]--[[package]]-name = "lalrpop-util"-version = "0.19.8"+name = "kuchiki"+version = "0.8.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bcf796c978e9b4d983414f4caedc9273aa33ee214c5b887bd55fde84c85d2dc4"+checksum = "1ea8e9c6e031377cff82ee3001dc8026cdf431ed4e2e6b51f98ab8c73484a358"dependencies = [- "regex",+ "cssparser",+ "html5ever",+ "matches",+ "selectors",][[package]]@@ -1471,12 +1418,32 @@ version = "0.9.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "74141c8af4bb8136dafb5705826bdd9dce823021db897c1129191804140ddf84"+[[package]]+name = "mac"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"+[[package]]name = "maplit"version = "1.0.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"+[[package]]+name = "markup5ever"+version = "0.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd"+dependencies = [+ "log",+ "phf",+ "phf_codegen",+ "string_cache",+ "string_cache_codegen",+ "tendril",+]+[[package]]name = "matchers"version = "0.1.0"@@ -1498,17 +1465,6 @@ version = "0.5.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"-[[package]]-name = "md-5"-version = "0.9.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15"-dependencies = [- "block-buffer",- "digest",- "opaque-debug",-]-[[package]]name = "md5"version = "0.7.0"@@ -1617,7 +1573,7 @@ version = "0.7.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"dependencies = [- "getrandom",+ "getrandom 0.2.6",][[package]]@@ -1644,6 +1600,12 @@ version = "1.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"+[[package]]+name = "nodrop"+version = "0.1.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"+[[package]]name = "nom"version = "7.1.1"@@ -1724,12 +1686,6 @@ dependencies = ["loom",]-[[package]]-name = "opaque-debug"-version = "0.3.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"-[[package]]name = "openssl"version = "0.10.40"@@ -1799,17 +1755,6 @@ dependencies = ["stable_deref_trait",]-[[package]]-name = "parking_lot"-version = "0.11.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"-dependencies = [- "instant",- "lock_api",- "parking_lot_core 0.8.5",-]-[[package]]name = "parking_lot"version = "0.12.1"@@ -1817,21 +1762,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"dependencies = ["lock_api",- "parking_lot_core 0.9.3",-]--[[package]]-name = "parking_lot_core"-version = "0.8.5"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"-dependencies = [- "cfg-if",- "instant",- "libc",- "redox_syscall",- "smallvec",- "winapi",+ "parking_lot_core",][[package]]@@ -1866,29 +1797,77 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"[[package]]-name = "petgraph"-version = "0.6.2"+name = "phf"+version = "0.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143"+checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"dependencies = [- "fixedbitset",- "indexmap",+ "phf_macros",+ "phf_shared 0.8.0",+ "proc-macro-hack",][[package]]-name = "phf_shared"+name = "phf_codegen"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815"+dependencies = [+ "phf_generator 0.8.0",+ "phf_shared 0.8.0",+]++[[package]]+name = "phf_generator"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"+dependencies = [+ "phf_shared 0.8.0",+ "rand 0.7.3",+]++[[package]]+name = "phf_generator"version = "0.10.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"+checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"+dependencies = [+ "phf_shared 0.10.0",+ "rand 0.8.5",+]++[[package]]+name = "phf_macros"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c"+dependencies = [+ "phf_generator 0.8.0",+ "phf_shared 0.8.0",+ "proc-macro-hack",+ "proc-macro2",+ "quote",+ "syn",+]++[[package]]+name = "phf_shared"+version = "0.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"dependencies = ["siphasher",][[package]]-name = "pico-args"-version = "0.4.2"+name = "phf_shared"+version = "0.10.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468"+checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"+dependencies = [+ "siphasher",+][[package]]name = "pin-project"@@ -1988,6 +1967,12 @@ dependencies = ["version_check",]+[[package]]+name = "proc-macro-hack"+version = "0.5.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"+[[package]]name = "proc-macro2"version = "1.0.39"@@ -2015,6 +2000,20 @@ dependencies = ["proc-macro2",]+[[package]]+name = "rand"+version = "0.7.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"+dependencies = [+ "getrandom 0.1.16",+ "libc",+ "rand_chacha 0.2.2",+ "rand_core 0.5.1",+ "rand_hc",+ "rand_pcg",+]+[[package]]name = "rand"version = "0.8.5"@@ -2022,8 +2021,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"dependencies = ["libc",- "rand_chacha",- "rand_core",+ "rand_chacha 0.3.1",+ "rand_core 0.6.3",+]++[[package]]+name = "rand_chacha"+version = "0.2.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"+dependencies = [+ "ppv-lite86",+ "rand_core 0.5.1",][[package]]@@ -2033,7 +2042,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"dependencies = ["ppv-lite86",- "rand_core",+ "rand_core 0.6.3",+]++[[package]]+name = "rand_core"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"+dependencies = [+ "getrandom 0.1.16",][[package]]@@ -2042,7 +2060,25 @@ version = "0.6.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"dependencies = [- "getrandom",+ "getrandom 0.2.6",+]++[[package]]+name = "rand_hc"+version = "0.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"+dependencies = [+ "rand_core 0.5.1",+]++[[package]]+name = "rand_pcg"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"+dependencies = [+ "rand_core 0.5.1",][[package]]@@ -2078,17 +2114,6 @@ dependencies = ["bitflags",]-[[package]]-name = "redox_users"-version = "0.4.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"-dependencies = [- "getrandom",- "redox_syscall",- "thiserror",-]-[[package]]name = "regex"version = "1.6.0"@@ -2176,88 +2201,6 @@ dependencies = ["librocksdb-sys",]-[[package]]-name = "rusoto_core"-version = "0.48.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2"-dependencies = [- "async-trait",- "base64",- "bytes",- "crc32fast",- "futures",- "http",- "hyper",- "hyper-tls",- "lazy_static",- "log",- "rusoto_credential",- "rusoto_signature",- "rustc_version",- "serde",- "serde_json",- "tokio",- "xml-rs",-]--[[package]]-name = "rusoto_credential"-version = "0.48.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05"-dependencies = [- "async-trait",- "chrono",- "dirs-next",- "futures",- "hyper",- "serde",- "serde_json",- "shlex",- "tokio",- "zeroize",-]--[[package]]-name = "rusoto_s3"-version = "0.48.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7aae4677183411f6b0b412d66194ef5403293917d66e70ab118f07cc24c5b14d"-dependencies = [- "async-trait",- "bytes",- "futures",- "rusoto_core",- "xml-rs",-]--[[package]]-name = "rusoto_signature"-version = "0.48.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272"-dependencies = [- "base64",- "bytes",- "chrono",- "digest",- "futures",- "hex",- "hmac",- "http",- "hyper",- "log",- "md-5",- "percent-encoding",- "pin-project-lite",- "rusoto_credential",- "rustc_version",- "serde",- "sha2",- "tokio",-]-[[package]]name = "rust-stemmers"version = "1.2.0"@@ -2346,6 +2289,26 @@ dependencies = ["libc",]+[[package]]+name = "selectors"+version = "0.22.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"+dependencies = [+ "bitflags",+ "cssparser",+ "derive_more",+ "fxhash",+ "log",+ "matches",+ "phf",+ "phf_codegen",+ "precomputed-hash",+ "servo_arc",+ "smallvec",+ "thin-slice",+]+[[package]]name = "semver"version = "1.0.13"@@ -2396,16 +2359,13 @@ dependencies = [][[package]]-name = "sha2"-version = "0.9.9"+name = "servo_arc"+version = "0.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"+checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432"dependencies = [- "block-buffer",- "cfg-if",- "cpufeatures",- "digest",- "opaque-debug",+ "nodrop",+ "stable_deref_trait",][[package]]@@ -2444,23 +2404,6 @@ version = "0.4.6"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32"-[[package]]-name = "sled"-version = "0.34.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"-dependencies = [- "crc32fast",- "crossbeam-epoch",- "crossbeam-utils",- "fs2",- "fxhash",- "libc",- "log",- "parking_lot 0.11.2",- "zstd",-]-[[package]]name = "smallvec"version = "1.8.0"@@ -2500,22 +2443,29 @@ checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08"dependencies = ["new_debug_unreachable","once_cell",- "parking_lot 0.12.1",- "phf_shared",+ "parking_lot",+ "phf_shared 0.10.0","precomputed-hash",+ "serde",][[package]]-name = "strsim"-version = "0.10.0"+name = "string_cache_codegen"+version = "0.5.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"+checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988"+dependencies = [+ "phf_generator 0.10.0",+ "phf_shared 0.10.0",+ "proc-macro2",+ "quote",+][[package]]-name = "subtle"-version = "2.4.1"+name = "strsim"+version = "0.10.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"[[package]]name = "syn"@@ -2638,14 +2588,14 @@ dependencies = [][[package]]-name = "term"-version = "0.7.0"+name = "tendril"+version = "0.4.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"+checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"dependencies = [- "dirs-next",- "rustversion",- "winapi",+ "futf",+ "mac",+ "utf-8",][[package]]@@ -2673,6 +2623,12 @@ version = "0.15.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"+[[package]]+name = "thin-slice"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"+[[package]]name = "thiserror"version = "1.0.31"@@ -2745,15 +2701,6 @@ dependencies = ["serde",]-[[package]]-name = "tiny-keccak"-version = "2.0.2"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"-dependencies = [- "crunchy",-]-[[package]]name = "tinyvec"version = "1.6.0"@@ -2781,7 +2728,7 @@ dependencies = ["mio","num_cpus","once_cell",- "parking_lot 0.12.1",+ "parking_lot","pin-project-lite","signal-hook-registry","socket2",@@ -2955,12 +2902,6 @@ version = "0.2.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"-[[package]]-name = "typenum"-version = "1.15.0"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"-[[package]]name = "unicase"version = "2.6.0"@@ -2991,12 +2932,6 @@ dependencies = ["tinyvec",]-[[package]]-name = "unicode-xid"-version = "0.2.3"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"-[[package]]name = "url"version = "2.2.2"@@ -3009,6 +2944,12 @@ dependencies = ["percent-encoding",]+[[package]]+name = "utf-8"+version = "0.7.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"+[[package]]name = "utf8-ranges"version = "1.0.5"@@ -3027,7 +2968,7 @@ version = "1.1.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f"dependencies = [- "getrandom",+ "getrandom 0.2.6","serde",]@@ -3059,6 +3000,12 @@ dependencies = ["try-lock",]+[[package]]+name = "wasi"+version = "0.9.0+wasi-snapshot-preview1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"+[[package]]name = "wasi"version = "0.10.0+wasi-snapshot-preview1"@@ -3246,37 +3193,6 @@ dependencies = ["winapi",]-[[package]]-name = "xml-rs"-version = "0.8.4"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3"--[[package]]-name = "zeroize"-version = "1.5.7"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f"--[[package]]-name = "zstd"-version = "0.9.2+zstd.1.5.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2390ea1bf6c038c39674f22d95f0564725fc06034a47129179810b2fc58caa54"-dependencies = [- "zstd-safe",-]--[[package]]-name = "zstd-safe"-version = "4.1.3+zstd.1.5.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e99d81b99fb3c2c2c794e3fe56c305c63d5173a16a46b5850b07c935ffc7db79"-dependencies = [- "libc",- "zstd-sys",-]-[[package]]name = "zstd-sys"version = "1.6.2+zstd.1.5.1"diff --git a/Cargo.toml b/Cargo.tomlindex 3e65e897..dc16d9b6 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -20,11 +20,8 @@ itertools = "0.10.3"lru = "0.7.6"rayon = "1.5.3"reqwest = { version = "0.11.10", features = ["blocking"] }-rusoto_core = "0.48.0"-rusoto_s3 = "0.48.0"serde = "1.0.137"serde_json = "1.0.81"-sled = { version = "0.34.7", features = ["compression"] }tantivy = "0.18.0"thiserror = "1.0.31"toml = "0.5.9"@@ -40,7 +37,6 @@ html-escape = "0.2.11"logos = "0.12.1"csv = "1.1.6"fst = { version = "0.4.7", features = ["levenshtein"] }-lazy_static = "1.4.0"rocksdb = "0.18.0"image = "0.24.3"chrono = "0.4.20"@@ -51,16 +47,13 @@ bzip2 = "0.4.3"parse_wiki_text = "0.1.5"md5 = "0.7.0"memmap = "0.7.0"-lalrpop-util = { version = "0.19.8", features = ["lexer"] }once_cell = "1.13.1"regex = "1.6.0"http = "0.2.8"-[dev-dependencies]+kuchiki = "0.8.1"+[dev-dependencies]maplit = "1.0.2"[profile.release]debug = true--[build-dependencies]-lalrpop = "0.19.8"diff --git a/build.rs b/build.rsindex 98dcc47d..9f383a77 100644--- a/build.rs+++ b/build.rs@@ -10,6 +10,4 @@ fn main() {println!("cargo:rerun-if-changed=yarn.lock");println!("cargo:rerun-if-changed=tailwind.config.js");println!("cargo:rerun-if-changed=postcss.config.js");-- lalrpop::process_root().unwrap();}diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 4be98569..9cca690d 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -154,7 +154,6 @@ impl Indexer {.collect();let job_config = match config.warc_source.clone() {- WarcSource::S3(_) => todo!("s3 not supported yet"),WarcSource::HTTP(config) => JobConfig::Http(config),WarcSource::Local(config) => JobConfig::Local(config),};@@ -195,7 +194,6 @@ impl Indexer {let warc_paths = config.warc_source.paths()?;let job_config = match config.warc_source.clone() {- WarcSource::S3(_) => todo!("s3 not supported yet"),WarcSource::HTTP(config) => JobConfig::Http(config),WarcSource::Local(config) => JobConfig::Local(config),};diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rsindex e3104592..e8349210 100644--- a/src/entrypoint/webgraph.rs+++ b/src/entrypoint/webgraph.rs@@ -124,7 +124,6 @@ impl WebgraphEntrypoint {.collect();let job_config = match config.warc_source.clone() {- WarcSource::S3(_) => todo!("s3 not supported yet"),WarcSource::HTTP(config) => JobConfig::Http(config),WarcSource::Local(config) => JobConfig::Local(config),};@@ -165,7 +164,6 @@ impl WebgraphEntrypoint {let warc_paths = config.warc_source.paths()?;let job_config = match config.warc_source.clone() {- WarcSource::S3(_) => todo!("s3 not supported yet"),WarcSource::HTTP(config) => JobConfig::Http(config),WarcSource::Local(config) => JobConfig::Local(config),};diff --git a/src/kv/mod.rs b/src/kv/mod.rsindex 71d4f542..32b6713d 100644--- a/src/kv/mod.rs+++ b/src/kv/mod.rs@@ -17,7 +17,6 @@use serde::{de::DeserializeOwned, Serialize};pub mod rocksdb_store;-pub mod sled_store;pub trait Kv<K, V>wherediff --git a/src/kv/sled_store.rs b/src/kv/sled_store.rsdeleted file mode 100644index b9d3c641..00000000--- a/src/kv/sled_store.rs+++ /dev/null@@ -1,79 +0,0 @@-// Cuely is an open source web search engine.-// Copyright (C) 2022 Cuely ApS-//-// This program is free software: you can redistribute it and/or modify-// it under the terms of the GNU Affero General Public License as-// published by the Free Software Foundation, either version 3 of the-// License, or (at your option) any later version.-//-// This program is distributed in the hope that it will be useful,-// but WITHOUT ANY WARRANTY; without even the implied warranty of-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the-// GNU Affero General Public License for more details.-//-// You should have received a copy of the GNU Affero General Public License-// along with this program. If not, see <https://www.gnu.org/licenses/>.--use std::marker::PhantomData;--use serde::{de::DeserializeOwned, Serialize};--use crate::kv::Kv;--pub struct SledStore {}--impl<K, V> Kv<K, V> for sled::Tree-where- K: Serialize + DeserializeOwned + 'static,- V: Serialize + DeserializeOwned + 'static,-{- fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {- self.get(key)- .expect("failed to retrieve key")- .map(|v| v.to_vec())- }-- fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>) {- self.insert(key, value).expect("failed to insert value");- }-- fn flush(&self) {- self.flush().expect("failed to flush");- }-- fn iter(&self) -> Box<dyn Iterator<Item = (K, V)>> {- Box::new(IntoIter {- inner: self.iter(),- key: Default::default(),- value: Default::default(),- })- }-}--pub struct IntoIter<K, V>-where- K: Serialize + DeserializeOwned,- V: Serialize + DeserializeOwned,-{- inner: sled::Iter,- key: PhantomData<K>,- value: PhantomData<V>,-}--impl<K, V> Iterator for IntoIter<K, V>-where- K: Serialize + DeserializeOwned,- V: Serialize + DeserializeOwned,-{- type Item = (K, V);-- fn next(&mut self) -> Option<Self::Item> {- self.inner.next().map(|res| {- let (key_bytes, value_bytes) = res.expect("Failed to get next record from sled tree");- (- bincode::deserialize(&key_bytes).expect("Failed to deserialize key"),- bincode::deserialize(&value_bytes).expect("Failed to deserialize value"),- )- })- }-}diff --git a/src/lib.rs b/src/lib.rsindex 0507e230..c33bca29 100644--- a/src/lib.rs+++ b/src/lib.rs@@ -82,7 +82,6 @@ pub struct WebgraphLocalConfig {#[derive(Debug, Serialize, Deserialize, Clone)]#[serde(tag = "type")]pub enum WarcSource {- S3(S3Config),HTTP(HttpConfig),Local(LocalConfig),}@@ -91,12 +90,6 @@ impl WarcSource {pub fn paths(&self) -> Result<Vec<String>> {let mut warc_paths = Vec::new();match &self {- WarcSource::S3(config) => {- let file = File::open(&config.warc_paths_file)?;- for line in io::BufReader::new(file).lines() {- warc_paths.push(line?);- }- }WarcSource::HTTP(config) => {let file = File::open(&config.warc_paths_file)?;for line in io::BufReader::new(file).lines() {@@ -118,14 +111,6 @@ pub struct LocalConfig {names: Vec<String>,}-#[derive(Debug, Serialize, Deserialize, Clone)]-pub struct S3Config {- name: String,- endpoint: String,- warc_paths_file: String,- bucket: String,-}-#[derive(Debug, Serialize, Deserialize, Clone)]pub struct HttpConfig {base_url: String,@@ -134,15 +119,9 @@ pub struct HttpConfig {#[derive(Error, Debug)]pub enum Error {- #[error("Failed to download object from S3")]- S3DownloadError,-#[error("Failed to download object from HTTP")]HTTPDownloadError(#[from] reqwest::Error),- #[error("Failed to get the object from S3")]- GetObjectError(#[from] rusoto_core::RusotoError<rusoto_s3::GetObjectError>),-#[error("Got an IO error")]IOError(#[from] io::Error),diff --git a/src/ranking/centrality_store.rs b/src/ranking/centrality_store.rsindex 59fc675b..5b89898f 100644--- a/src/ranking/centrality_store.rs+++ b/src/ranking/centrality_store.rs@@ -16,42 +16,25 @@use std::path::Path;+use crate::kv::{rocksdb_store::RocksDbStore, Kv};+pub struct CentralityStore {- inner: sled::Tree,+ inner: Box<dyn Kv<String, f64>>,}impl CentralityStore {pub fn new<P: AsRef<Path>>(path: P) -> Self {- let db = sled::Config::default()- .path(path)- .use_compression(true)- .mode(sled::Mode::LowSpace)- .open()- .expect("Failed to open database");-Self {- inner: db- .open_tree("centrality_store")- .expect("Failed to open tree"),+ inner: RocksDbStore::open(path),}}pub fn insert(&mut self, key: String, centrality: f64) {- let key_bytes = bincode::serialize(&key).expect("Failed to serialize key");- let centrality_bytes =- bincode::serialize(¢rality).expect("Failed to serialize centrality");-- self.inner- .insert(key_bytes, centrality_bytes)- .expect("Failed to insert into tree");+ self.inner.insert(key, centrality);}pub fn get(&self, key: &str) -> Option<f64> {- let key_bytes = bincode::serialize(key).expect("Failed to serialize key");- self.inner- .get(key_bytes)- .expect("Failed to retrieve value from key")- .map(|bytes| bincode::deserialize(&bytes).expect("Failed to deserialize value"))+ self.inner.get(&key.to_string())}pub fn append(&mut self, it: impl Iterator<Item = (String, f64)>) {@@ -59,6 +42,7 @@ impl CentralityStore {.for_each(|(key, value)| {self.insert(key, value);});- self.inner.flush().unwrap();++ self.inner.flush();}}diff --git a/src/warc.rs b/src/warc.rsindex 32d48930..9e174bc6 100644--- a/src/warc.rs+++ b/src/warc.rs@@ -71,12 +71,6 @@ impl WarcFile {pub(crate) fn download(source: WarcSource, path: &str) -> Result<Self> {for dur in ExponentialBackoff::from_millis(10).take(5) {let res = match source.clone() {- WarcSource::S3(config) => WarcFile::download_from_s3(- path,- config.name.clone(),- config.endpoint.clone(),- config.bucket,- ),WarcSource::HTTP(config) => WarcFile::download_from_http(path, config.base_url),WarcSource::Local(config) => WarcFile::load_from_folder(path, &config.folder),};@@ -122,39 +116,6 @@ impl WarcFile {Ok(WarcFile::new(bytes))}-- fn download_from_s3(- _path: &str,- _region_name: String,- _region_endpoint: String,- _bucket: String,- ) -> Result<Self> {- todo!();- // let path = path.to_string();- // let region = Region::Custom {- // name: region_name,- // endpoint: region_endpoint,- // };-- // let client = S3Client::new(region);-- // let obj = client- // .get_object(GetObjectRequest {- // bucket,- // key: path,- // ..Default::default()- // })- // .await?;-- // let mut bytes = Vec::new();- // obj.body- // .ok_or(Error::S3DownloadError)?- // .into_async_read()- // .read_to_end(&mut bytes)- // .await?;-- // Ok(WarcFile::new(bytes))- }}#[derive(Debug)]diff --git a/src/webgraph/graph_store.rs b/src/webgraph/graph_store.rsindex e9c483c6..1f4b6ad1 100644--- a/src/webgraph/graph_store.rs+++ b/src/webgraph/graph_store.rs@@ -22,7 +22,7 @@ use lru::LruCache;use serde::{de::DeserializeOwned, Serialize};use super::{Edge, EdgeIterator, Node, NodeID, Store, StoredEdge};-use crate::kv::{rocksdb_store::RocksDbStore, sled_store::SledStore, Kv};+use crate::kv::{rocksdb_store::RocksDbStore, Kv};pub(crate) struct Adjacency {pub(crate) tree: BlockedCachedTree<NodeID, Vec<StoredEdge>>,}@@ -314,47 +314,6 @@ impl Store for RocksDbStore {}}-impl Store for SledStore {- fn open<P: AsRef<std::path::Path>>(path: P) -> GraphStore<Self> {- let db = sled::Config::default()- .path(path)- .use_compression(true)- .mode(sled::Mode::LowSpace)- .open()- .expect("Failed to open database");-- let adjacency = Box::new(db.open_tree("adjacency").expect("unable to open sled tree"));- let reversed_adjacency = Box::new(- db.open_tree("reversed_adjacency")- .expect("unable to open sled tree"),- );- let node2id = Box::new(db.open_tree("node2id").expect("unable to open sled tree"));- let id2node = Box::new(db.open_tree("id2node").expect("unable to open sled tree"));- let meta = Box::new(db.open_tree("meta").expect("unable to open sled tree"));-- GraphStore {- adjacency: RefCell::new(Adjacency {- tree: BlockedCachedTree {- inner: CachedTree::new(adjacency, 10_000),- block_size: 1_024,- },- }),- reversed_adjacency: RefCell::new(Adjacency {- tree: BlockedCachedTree {- inner: CachedTree::new(reversed_adjacency, 10_000),- block_size: 1_024,- },- }),- node2id: RefCell::new(CachedTree::new(node2id, 100_000)),- id2node: RefCell::new(BlockedCachedTree {- inner: CachedTree::new(id2node, 100_000),- block_size: 1_024,- }),- meta: RefCell::new(CachedTree::new(meta, 1_000)),- store: Default::default(),- }- }-}#[cfg(test)]mod test {use super::*;diff --git a/src/webpage/just_text.rs b/src/webpage/just_text.rsindex b12f8545..9e479ad1 100644--- a/src/webpage/just_text.rs+++ b/src/webpage/just_text.rs@@ -16,7 +16,6 @@use std::collections::{HashMap, HashSet};-use lazy_static::lazy_static;use logos::{Lexer, Logos};use whatlang::Lang;@@ -70,8 +69,8 @@ macro_rules! include_stopwords {}};}-lazy_static! {- static ref STOPWORDS: HashMap<Lang, HashSet<String>> = {+static STOPWORDS: once_cell::sync::Lazy<HashMap<Lang, HashSet<String>>> =+ once_cell::sync::Lazy::new(|| {include_stopwords!("../../stopwords/Afrikaans.txt" => Lang::Afr,"../../stopwords/Arabic.txt" => Lang::Ara,@@ -120,8 +119,7 @@ lazy_static! {"../../stopwords/Spanish.txt" => Lang::Spa,"../../stopwords/Japanese.txt" => Lang::Jpn)- };-}+ });pub struct JustText {pub max_link_density: f64,