Skip to content

web search done right

dependency cleanup

Mikkel Denker5c35f5d10ccc
diff --git a/Cargo.lock b/Cargo.lock
index 7655040e..6a1925ef 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -20,7 +20,7 @@ version = "0.7.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47"
dependencies = [
- "getrandom",
+ "getrandom 0.2.6",
"once_cell",
"version_check",
]
@@ -49,15 +49,6 @@ version = "1.0.58"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bb07d2053ccdbe10e2af2995a2f116c1330396493dc1269f6a91d0ae82e19704"
-[[package]]
-name = "ascii-canvas"
-version = "3.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8824ecca2e851cec16968d54a01dd372ef8f95b244fb84b84e70128be347c3c6"
-dependencies = [
- "term",
-]
-
[[package]]
name = "askama"
version = "0.11.1"
@@ -237,21 +228,6 @@ dependencies = [
"shlex",
]
-[[package]]
-name = "bit-set"
-version = "0.5.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0700ddab506f33b20a03b13996eccd309a48e5ff77d0d95926aa0210fb4e95f1"
-dependencies = [
- "bit-vec",
-]
-
-[[package]]
-name = "bit-vec"
-version = "0.6.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb"
-
[[package]]
name = "bit_field"
version = "0.10.1"
@@ -273,15 +249,6 @@ dependencies = [
"crunchy",
]
-[[package]]
-name = "block-buffer"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4152116fd6e9dadb291ae18fc1ec3575ed6d84c29642d97890f4b4a3417297e4"
-dependencies = [
- "generic-array",
-]
-
[[package]]
name = "bstr"
version = "0.2.17"
@@ -378,7 +345,6 @@ dependencies = [
"js-sys",
"num-integer",
"num-traits",
- "serde",
"time 0.1.44",
"wasm-bindgen",
"winapi",
@@ -462,6 +428,12 @@ dependencies = [
"winapi",
]
+[[package]]
+name = "convert_case"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
+
[[package]]
name = "core-foundation"
version = "0.9.3"
@@ -478,15 +450,6 @@ version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5827cebf4670468b8772dd191856768aedcb1b0278a04f989f7766351917b9dc"
-[[package]]
-name = "cpufeatures"
-version = "0.2.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "59a6001667ab124aebae2a495118e11d30984c3a653e99d86d58971708cf5e4b"
-dependencies = [
- "libc",
-]
-
[[package]]
name = "crc32fast"
version = "1.3.2"
@@ -548,13 +511,30 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7a81dae078cea95a014a339291cec439d2f232ebe854a9d672b796c6afafa9b7"
[[package]]
-name = "crypto-mac"
-version = "0.11.1"
+name = "cssparser"
+version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1d1a86f49236c215f271d40892d5fc950490551400b02ef360692c29815c714"
+checksum = "754b69d351cdc2d8ee09ae203db831e005560fc6030da058f86ad60c92a9cb0a"
dependencies = [
- "generic-array",
- "subtle",
+ "cssparser-macros",
+ "dtoa-short",
+ "itoa 0.4.8",
+ "matches",
+ "phf",
+ "proc-macro2",
+ "quote",
+ "smallvec",
+ "syn",
+]
+
+[[package]]
+name = "cssparser-macros"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfae75de57f2b2e85e8768c3ea840fd159c8f33e2b6522c7835b7abac81be16e"
+dependencies = [
+ "quote",
+ "syn",
]
[[package]]
@@ -611,9 +591,7 @@ dependencies = [
"image",
"indicatif",
"itertools",
- "lalrpop",
- "lalrpop-util",
- "lazy_static",
+ "kuchiki",
"logos",
"lru",
"maplit",
@@ -626,11 +604,8 @@ dependencies = [
"regex",
"reqwest",
"rocksdb",
- "rusoto_core",
- "rusoto_s3",
"serde",
"serde_json",
- "sled",
"tantivy",
"thiserror",
"tokio",
@@ -650,6 +625,19 @@ dependencies = [
"adler32",
]
+[[package]]
+name = "derive_more"
+version = "0.99.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321"
+dependencies = [
+ "convert_case",
+ "proc-macro2",
+ "quote",
+ "rustc_version",
+ "syn",
+]
+
[[package]]
name = "diff"
version = "0.1.12"
@@ -657,56 +645,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0e25ea47919b1560c4e3b7fe0aaab9becf5b84a10325ddf7db0f0ba5e1026499"
[[package]]
-name = "digest"
-version = "0.9.0"
+name = "downcast-rs"
+version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3dd60d1080a57a05ab032377049e0591415d2b31afd7028356dbf3cc6dcb066"
-dependencies = [
- "generic-array",
-]
+checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"
[[package]]
-name = "dirs-next"
-version = "2.0.0"
+name = "dtoa"
+version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b98cf8ebf19c3d1b223e151f99a4f9f0690dca41414773390fc824184ac833e1"
-dependencies = [
- "cfg-if",
- "dirs-sys-next",
-]
+checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0"
[[package]]
-name = "dirs-sys-next"
-version = "0.1.2"
+name = "dtoa-short"
+version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4ebda144c4fe02d1f7ea1a7d9641b6fc6b580adcfa024ae48797ecdeb6825b4d"
+checksum = "bde03329ae10e79ede66c9ce4dc930aa8599043b0743008548680f25b91502d6"
dependencies = [
- "libc",
- "redox_users",
- "winapi",
+ "dtoa",
]
-[[package]]
-name = "downcast-rs"
-version = "1.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9ea835d29036a4087793836fa931b08837ad5e957da9e23886b29586fb9b6650"
-
[[package]]
name = "either"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e78d4f1cc4ae33bbfc157ed5d5a5ef3bc29227303d595861deb238fcec4e9457"
-[[package]]
-name = "ena"
-version = "0.14.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7402b94a93c24e742487327a7cd839dc9d36fec9de9fb25b09f2dae459f36c3"
-dependencies = [
- "log",
-]
-
[[package]]
name = "encode_unicode"
version = "0.3.6"
@@ -746,7 +710,7 @@ checksum = "ec3245a0ca564e7f3c797d20d833a6870f57a728ac967d5225b3ffdef4465011"
dependencies = [
"lazy_static",
"log",
- "rand",
+ "rand 0.8.5",
]
[[package]]
@@ -774,12 +738,6 @@ dependencies = [
"instant",
]
-[[package]]
-name = "fixedbitset"
-version = "0.4.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
-
[[package]]
name = "flate2"
version = "1.0.24"
@@ -853,6 +811,16 @@ dependencies = [
"utf8-ranges",
]
+[[package]]
+name = "futf"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
+dependencies = [
+ "mac",
+ "new_debug_unreachable",
+]
+
[[package]]
name = "futures"
version = "0.3.21"
@@ -965,13 +933,14 @@ dependencies = [
]
[[package]]
-name = "generic-array"
-version = "0.14.6"
+name = "getrandom"
+version = "0.1.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bff49e947297f3312447abdca79f45f4738097cc82b06e72054d2223f601f1b9"
+checksum = "8fc3cb4d91f53b50155bdcfd23f6a4c39ae1969c2ae85982b135750cccaf5fce"
dependencies = [
- "typenum",
- "version_check",
+ "cfg-if",
+ "libc",
+ "wasi 0.9.0+wasi-snapshot-preview1",
]
[[package]]
@@ -1062,28 +1031,26 @@ dependencies = [
]
[[package]]
-name = "hex"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
-
-[[package]]
-name = "hmac"
-version = "0.11.0"
+name = "html-escape"
+version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2a2a2320eb7ec0ebe8da8f744d7812d9fc4cb4d09344ac01898dbcb6a20ae69b"
+checksum = "b8e7479fa1ef38eb49fb6a42c426be515df2d063f06cb8efd3e50af073dbc26c"
dependencies = [
- "crypto-mac",
- "digest",
+ "utf8-width",
]
[[package]]
-name = "html-escape"
-version = "0.2.11"
+name = "html5ever"
+version = "0.25.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b8e7479fa1ef38eb49fb6a42c426be515df2d063f06cb8efd3e50af073dbc26c"
+checksum = "e5c13fb08e5d4dfc151ee5e88bae63f7773d61852f3bdc73c9f4b9e1bde03148"
dependencies = [
- "utf8-width",
+ "log",
+ "mac",
+ "markup5ever",
+ "proc-macro2",
+ "quote",
+ "syn",
]
[[package]]
@@ -1304,35 +1271,15 @@ dependencies = [
]
[[package]]
-name = "lalrpop"
-version = "0.19.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b30455341b0e18f276fa64540aff54deafb54c589de6aca68659c63dd2d5d823"
-dependencies = [
- "ascii-canvas",
- "atty",
- "bit-set",
- "diff",
- "ena",
- "itertools",
- "lalrpop-util",
- "petgraph",
- "pico-args",
- "regex",
- "regex-syntax 0.6.27",
- "string_cache",
- "term",
- "tiny-keccak",
- "unicode-xid",
-]
-
-[[package]]
-name = "lalrpop-util"
-version = "0.19.8"
+name = "kuchiki"
+version = "0.8.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bcf796c978e9b4d983414f4caedc9273aa33ee214c5b887bd55fde84c85d2dc4"
+checksum = "1ea8e9c6e031377cff82ee3001dc8026cdf431ed4e2e6b51f98ab8c73484a358"
dependencies = [
- "regex",
+ "cssparser",
+ "html5ever",
+ "matches",
+ "selectors",
]
[[package]]
@@ -1471,12 +1418,32 @@ version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "74141c8af4bb8136dafb5705826bdd9dce823021db897c1129191804140ddf84"
+[[package]]
+name = "mac"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
+
[[package]]
name = "maplit"
version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d"
+[[package]]
+name = "markup5ever"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a24f40fb03852d1cdd84330cddcaf98e9ec08a7b7768e952fad3b4cf048ec8fd"
+dependencies = [
+ "log",
+ "phf",
+ "phf_codegen",
+ "string_cache",
+ "string_cache_codegen",
+ "tendril",
+]
+
[[package]]
name = "matchers"
version = "0.1.0"
@@ -1498,17 +1465,6 @@ version = "0.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "73cbba799671b762df5a175adf59ce145165747bb891505c43d09aefbbf38beb"
-[[package]]
-name = "md-5"
-version = "0.9.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b5a279bb9607f9f53c22d496eade00d138d1bdcccd07d74650387cf94942a15"
-dependencies = [
- "block-buffer",
- "digest",
- "opaque-debug",
-]
-
[[package]]
name = "md5"
version = "0.7.0"
@@ -1617,7 +1573,7 @@ version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
dependencies = [
- "getrandom",
+ "getrandom 0.2.6",
]
[[package]]
@@ -1644,6 +1600,12 @@ version = "1.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54"
+[[package]]
+name = "nodrop"
+version = "0.1.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72ef4a56884ca558e5ddb05a1d1e7e1bfd9a68d9ed024c21704cc98872dae1bb"
+
[[package]]
name = "nom"
version = "7.1.1"
@@ -1724,12 +1686,6 @@ dependencies = [
"loom",
]
-[[package]]
-name = "opaque-debug"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "624a8340c38c1b80fd549087862da4ba43e08858af025b236e509b6649fc13d5"
-
[[package]]
name = "openssl"
version = "0.10.40"
@@ -1799,17 +1755,6 @@ dependencies = [
"stable_deref_trait",
]
-[[package]]
-name = "parking_lot"
-version = "0.11.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99"
-dependencies = [
- "instant",
- "lock_api",
- "parking_lot_core 0.8.5",
-]
-
[[package]]
name = "parking_lot"
version = "0.12.1"
@@ -1817,21 +1762,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f"
dependencies = [
"lock_api",
- "parking_lot_core 0.9.3",
-]
-
-[[package]]
-name = "parking_lot_core"
-version = "0.8.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d76e8e1493bcac0d2766c42737f34458f1c8c50c0d23bcb24ea953affb273216"
-dependencies = [
- "cfg-if",
- "instant",
- "libc",
- "redox_syscall",
- "smallvec",
- "winapi",
+ "parking_lot_core",
]
[[package]]
@@ -1866,29 +1797,77 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"
[[package]]
-name = "petgraph"
-version = "0.6.2"
+name = "phf"
+version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e6d5014253a1331579ce62aa67443b4a658c5e7dd03d4bc6d302b94474888143"
+checksum = "3dfb61232e34fcb633f43d12c58f83c1df82962dcdfa565a4e866ffc17dafe12"
dependencies = [
- "fixedbitset",
- "indexmap",
+ "phf_macros",
+ "phf_shared 0.8.0",
+ "proc-macro-hack",
]
[[package]]
-name = "phf_shared"
+name = "phf_codegen"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cbffee61585b0411840d3ece935cce9cb6321f01c45477d30066498cd5e1a815"
+dependencies = [
+ "phf_generator 0.8.0",
+ "phf_shared 0.8.0",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "17367f0cc86f2d25802b2c26ee58a7b23faeccf78a396094c13dced0d0182526"
+dependencies = [
+ "phf_shared 0.8.0",
+ "rand 0.7.3",
+]
+
+[[package]]
+name = "phf_generator"
version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
+checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
+dependencies = [
+ "phf_shared 0.10.0",
+ "rand 0.8.5",
+]
+
+[[package]]
+name = "phf_macros"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f6fde18ff429ffc8fe78e2bf7f8b7a5a5a6e2a8b58bc5a9ac69198bbda9189c"
+dependencies = [
+ "phf_generator 0.8.0",
+ "phf_shared 0.8.0",
+ "proc-macro-hack",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c00cf8b9eafe68dde5e9eaa2cef8ee84a9336a47d566ec55ca16589633b65af7"
dependencies = [
"siphasher",
]
[[package]]
-name = "pico-args"
-version = "0.4.2"
+name = "phf_shared"
+version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db8bcd96cb740d03149cbad5518db9fd87126a10ab519c011893b1754134c468"
+checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
+dependencies = [
+ "siphasher",
+]
[[package]]
name = "pin-project"
@@ -1988,6 +1967,12 @@ dependencies = [
"version_check",
]
+[[package]]
+name = "proc-macro-hack"
+version = "0.5.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5"
+
[[package]]
name = "proc-macro2"
version = "1.0.39"
@@ -2015,6 +2000,20 @@ dependencies = [
"proc-macro2",
]
+[[package]]
+name = "rand"
+version = "0.7.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
+dependencies = [
+ "getrandom 0.1.16",
+ "libc",
+ "rand_chacha 0.2.2",
+ "rand_core 0.5.1",
+ "rand_hc",
+ "rand_pcg",
+]
+
[[package]]
name = "rand"
version = "0.8.5"
@@ -2022,8 +2021,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
dependencies = [
"libc",
- "rand_chacha",
- "rand_core",
+ "rand_chacha 0.3.1",
+ "rand_core 0.6.3",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.2.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402"
+dependencies = [
+ "ppv-lite86",
+ "rand_core 0.5.1",
]
[[package]]
@@ -2033,7 +2042,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
dependencies = [
"ppv-lite86",
- "rand_core",
+ "rand_core 0.6.3",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
+dependencies = [
+ "getrandom 0.1.16",
]
[[package]]
@@ -2042,7 +2060,25 @@ version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d34f1408f55294453790c48b2f1ebbb1c5b4b7563eb1f418bcfcfdbb06ebb4e7"
dependencies = [
- "getrandom",
+ "getrandom 0.2.6",
+]
+
+[[package]]
+name = "rand_hc"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c"
+dependencies = [
+ "rand_core 0.5.1",
+]
+
+[[package]]
+name = "rand_pcg"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "16abd0c1b639e9eb4d7c50c0b8100b0d0f849be2349829c740fe8e6eb4816429"
+dependencies = [
+ "rand_core 0.5.1",
]
[[package]]
@@ -2078,17 +2114,6 @@ dependencies = [
"bitflags",
]
-[[package]]
-name = "redox_users"
-version = "0.4.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b033d837a7cf162d7993aded9304e30a83213c648b6e389db233191f891e5c2b"
-dependencies = [
- "getrandom",
- "redox_syscall",
- "thiserror",
-]
-
[[package]]
name = "regex"
version = "1.6.0"
@@ -2176,88 +2201,6 @@ dependencies = [
"librocksdb-sys",
]
-[[package]]
-name = "rusoto_core"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1db30db44ea73551326269adcf7a2169428a054f14faf9e1768f2163494f2fa2"
-dependencies = [
- "async-trait",
- "base64",
- "bytes",
- "crc32fast",
- "futures",
- "http",
- "hyper",
- "hyper-tls",
- "lazy_static",
- "log",
- "rusoto_credential",
- "rusoto_signature",
- "rustc_version",
- "serde",
- "serde_json",
- "tokio",
- "xml-rs",
-]
-
-[[package]]
-name = "rusoto_credential"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ee0a6c13db5aad6047b6a44ef023dbbc21a056b6dab5be3b79ce4283d5c02d05"
-dependencies = [
- "async-trait",
- "chrono",
- "dirs-next",
- "futures",
- "hyper",
- "serde",
- "serde_json",
- "shlex",
- "tokio",
- "zeroize",
-]
-
-[[package]]
-name = "rusoto_s3"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7aae4677183411f6b0b412d66194ef5403293917d66e70ab118f07cc24c5b14d"
-dependencies = [
- "async-trait",
- "bytes",
- "futures",
- "rusoto_core",
- "xml-rs",
-]
-
-[[package]]
-name = "rusoto_signature"
-version = "0.48.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a5ae95491c8b4847931e291b151127eccd6ff8ca13f33603eb3d0035ecb05272"
-dependencies = [
- "base64",
- "bytes",
- "chrono",
- "digest",
- "futures",
- "hex",
- "hmac",
- "http",
- "hyper",
- "log",
- "md-5",
- "percent-encoding",
- "pin-project-lite",
- "rusoto_credential",
- "rustc_version",
- "serde",
- "sha2",
- "tokio",
-]
-
[[package]]
name = "rust-stemmers"
version = "1.2.0"
@@ -2346,6 +2289,26 @@ dependencies = [
"libc",
]
+[[package]]
+name = "selectors"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df320f1889ac4ba6bc0cdc9c9af7af4bd64bb927bccdf32d81140dc1f9be12fe"
+dependencies = [
+ "bitflags",
+ "cssparser",
+ "derive_more",
+ "fxhash",
+ "log",
+ "matches",
+ "phf",
+ "phf_codegen",
+ "precomputed-hash",
+ "servo_arc",
+ "smallvec",
+ "thin-slice",
+]
+
[[package]]
name = "semver"
version = "1.0.13"
@@ -2396,16 +2359,13 @@ dependencies = [
]
[[package]]
-name = "sha2"
-version = "0.9.9"
+name = "servo_arc"
+version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d58a1e1bf39749807d89cf2d98ac2dfa0ff1cb3faa38fbb64dd88ac8013d800"
+checksum = "d98238b800e0d1576d8b6e3de32827c2d74bee68bb97748dcf5071fb53965432"
dependencies = [
- "block-buffer",
- "cfg-if",
- "cpufeatures",
- "digest",
- "opaque-debug",
+ "nodrop",
+ "stable_deref_trait",
]
[[package]]
@@ -2444,23 +2404,6 @@ version = "0.4.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eb703cfe953bccee95685111adeedb76fabe4e97549a58d16f03ea7b9367bb32"
-[[package]]
-name = "sled"
-version = "0.34.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7f96b4737c2ce5987354855aed3797279def4ebf734436c6aa4552cf8e169935"
-dependencies = [
- "crc32fast",
- "crossbeam-epoch",
- "crossbeam-utils",
- "fs2",
- "fxhash",
- "libc",
- "log",
- "parking_lot 0.11.2",
- "zstd",
-]
-
[[package]]
name = "smallvec"
version = "1.8.0"
@@ -2500,22 +2443,29 @@ checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08"
dependencies = [
"new_debug_unreachable",
"once_cell",
- "parking_lot 0.12.1",
- "phf_shared",
+ "parking_lot",
+ "phf_shared 0.10.0",
"precomputed-hash",
+ "serde",
]
[[package]]
-name = "strsim"
-version = "0.10.0"
+name = "string_cache_codegen"
+version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988"
+dependencies = [
+ "phf_generator 0.10.0",
+ "phf_shared 0.10.0",
+ "proc-macro2",
+ "quote",
+]
[[package]]
-name = "subtle"
-version = "2.4.1"
+name = "strsim"
+version = "0.10.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
[[package]]
name = "syn"
@@ -2638,14 +2588,14 @@ dependencies = [
]
[[package]]
-name = "term"
-version = "0.7.0"
+name = "tendril"
+version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c59df8ac95d96ff9bede18eb7300b0fda5e5d8d90960e76f8e14ae765eedbf1f"
+checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
dependencies = [
- "dirs-next",
- "rustversion",
- "winapi",
+ "futf",
+ "mac",
+ "utf-8",
]
[[package]]
@@ -2673,6 +2623,12 @@ version = "0.15.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
+[[package]]
+name = "thin-slice"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8eaa81235c7058867fa8c0e7314f33dcce9c215f535d1913822a2b3f5e289f3c"
+
[[package]]
name = "thiserror"
version = "1.0.31"
@@ -2745,15 +2701,6 @@ dependencies = [
"serde",
]
-[[package]]
-name = "tiny-keccak"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237"
-dependencies = [
- "crunchy",
-]
-
[[package]]
name = "tinyvec"
version = "1.6.0"
@@ -2781,7 +2728,7 @@ dependencies = [
"mio",
"num_cpus",
"once_cell",
- "parking_lot 0.12.1",
+ "parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
@@ -2955,12 +2902,6 @@ version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642"
-[[package]]
-name = "typenum"
-version = "1.15.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dcf81ac59edc17cc8697ff311e8f5ef2d99fcbd9817b34cec66f90b6c3dfd987"
-
[[package]]
name = "unicase"
version = "2.6.0"
@@ -2991,12 +2932,6 @@ dependencies = [
"tinyvec",
]
-[[package]]
-name = "unicode-xid"
-version = "0.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"
-
[[package]]
name = "url"
version = "2.2.2"
@@ -3009,6 +2944,12 @@ dependencies = [
"percent-encoding",
]
+[[package]]
+name = "utf-8"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
+
[[package]]
name = "utf8-ranges"
version = "1.0.5"
@@ -3027,7 +2968,7 @@ version = "1.1.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dd6469f4314d5f1ffec476e05f17cc9a78bc7a27a6a857842170bdf8d6f98d2f"
dependencies = [
- "getrandom",
+ "getrandom 0.2.6",
"serde",
]
@@ -3059,6 +3000,12 @@ dependencies = [
"try-lock",
]
+[[package]]
+name = "wasi"
+version = "0.9.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519"
+
[[package]]
name = "wasi"
version = "0.10.0+wasi-snapshot-preview1"
@@ -3246,37 +3193,6 @@ dependencies = [
"winapi",
]
-[[package]]
-name = "xml-rs"
-version = "0.8.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d2d7d3948613f75c98fd9328cfdcc45acc4d360655289d0a7d4ec931392200a3"
-
-[[package]]
-name = "zeroize"
-version = "1.5.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c394b5bd0c6f669e7275d9c20aa90ae064cb22e75a1cad54e1b34088034b149f"
-
-[[package]]
-name = "zstd"
-version = "0.9.2+zstd.1.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2390ea1bf6c038c39674f22d95f0564725fc06034a47129179810b2fc58caa54"
-dependencies = [
- "zstd-safe",
-]
-
-[[package]]
-name = "zstd-safe"
-version = "4.1.3+zstd.1.5.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e99d81b99fb3c2c2c794e3fe56c305c63d5173a16a46b5850b07c935ffc7db79"
-dependencies = [
- "libc",
- "zstd-sys",
-]
-
[[package]]
name = "zstd-sys"
version = "1.6.2+zstd.1.5.1"
diff --git a/Cargo.toml b/Cargo.toml
index 3e65e897..dc16d9b6 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -20,11 +20,8 @@ itertools = "0.10.3"
lru = "0.7.6"
rayon = "1.5.3"
reqwest = { version = "0.11.10", features = ["blocking"] }
-rusoto_core = "0.48.0"
-rusoto_s3 = "0.48.0"
serde = "1.0.137"
serde_json = "1.0.81"
-sled = { version = "0.34.7", features = ["compression"] }
tantivy = "0.18.0"
thiserror = "1.0.31"
toml = "0.5.9"
@@ -40,7 +37,6 @@ html-escape = "0.2.11"
logos = "0.12.1"
csv = "1.1.6"
fst = { version = "0.4.7", features = ["levenshtein"] }
-lazy_static = "1.4.0"
rocksdb = "0.18.0"
image = "0.24.3"
chrono = "0.4.20"
@@ -51,16 +47,13 @@ bzip2 = "0.4.3"
parse_wiki_text = "0.1.5"
md5 = "0.7.0"
memmap = "0.7.0"
-lalrpop-util = { version = "0.19.8", features = ["lexer"] }
once_cell = "1.13.1"
regex = "1.6.0"
http = "0.2.8"
-[dev-dependencies]
+kuchiki = "0.8.1"
+[dev-dependencies]
maplit = "1.0.2"
[profile.release]
debug = true
-
-[build-dependencies]
-lalrpop = "0.19.8"
diff --git a/build.rs b/build.rs
index 98dcc47d..9f383a77 100644
--- a/build.rs
+++ b/build.rs
@@ -10,6 +10,4 @@ fn main() {
println!("cargo:rerun-if-changed=yarn.lock");
println!("cargo:rerun-if-changed=tailwind.config.js");
println!("cargo:rerun-if-changed=postcss.config.js");
-
- lalrpop::process_root().unwrap();
}
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 4be98569..9cca690d 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -154,7 +154,6 @@ impl Indexer {
.collect();
let job_config = match config.warc_source.clone() {
- WarcSource::S3(_) => todo!("s3 not supported yet"),
WarcSource::HTTP(config) => JobConfig::Http(config),
WarcSource::Local(config) => JobConfig::Local(config),
};
@@ -195,7 +194,6 @@ impl Indexer {
let warc_paths = config.warc_source.paths()?;
let job_config = match config.warc_source.clone() {
- WarcSource::S3(_) => todo!("s3 not supported yet"),
WarcSource::HTTP(config) => JobConfig::Http(config),
WarcSource::Local(config) => JobConfig::Local(config),
};
diff --git a/src/entrypoint/webgraph.rs b/src/entrypoint/webgraph.rs
index e3104592..e8349210 100644
--- a/src/entrypoint/webgraph.rs
+++ b/src/entrypoint/webgraph.rs
@@ -124,7 +124,6 @@ impl WebgraphEntrypoint {
.collect();
let job_config = match config.warc_source.clone() {
- WarcSource::S3(_) => todo!("s3 not supported yet"),
WarcSource::HTTP(config) => JobConfig::Http(config),
WarcSource::Local(config) => JobConfig::Local(config),
};
@@ -165,7 +164,6 @@ impl WebgraphEntrypoint {
let warc_paths = config.warc_source.paths()?;
let job_config = match config.warc_source.clone() {
- WarcSource::S3(_) => todo!("s3 not supported yet"),
WarcSource::HTTP(config) => JobConfig::Http(config),
WarcSource::Local(config) => JobConfig::Local(config),
};
diff --git a/src/kv/mod.rs b/src/kv/mod.rs
index 71d4f542..32b6713d 100644
--- a/src/kv/mod.rs
+++ b/src/kv/mod.rs
@@ -17,7 +17,6 @@
use serde::{de::DeserializeOwned, Serialize};
pub mod rocksdb_store;
-pub mod sled_store;
pub trait Kv<K, V>
where
diff --git a/src/kv/sled_store.rs b/src/kv/sled_store.rs
deleted file mode 100644
index b9d3c641..00000000
--- a/src/kv/sled_store.rs
+++ /dev/null
@@ -1,79 +0,0 @@
-// Cuely is an open source web search engine.
-// Copyright (C) 2022 Cuely ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::marker::PhantomData;
-
-use serde::{de::DeserializeOwned, Serialize};
-
-use crate::kv::Kv;
-
-pub struct SledStore {}
-
-impl<K, V> Kv<K, V> for sled::Tree
-where
- K: Serialize + DeserializeOwned + 'static,
- V: Serialize + DeserializeOwned + 'static,
-{
- fn get_raw(&self, key: &[u8]) -> Option<Vec<u8>> {
- self.get(key)
- .expect("failed to retrieve key")
- .map(|v| v.to_vec())
- }
-
- fn insert_raw(&self, key: Vec<u8>, value: Vec<u8>) {
- self.insert(key, value).expect("failed to insert value");
- }
-
- fn flush(&self) {
- self.flush().expect("failed to flush");
- }
-
- fn iter(&self) -> Box<dyn Iterator<Item = (K, V)>> {
- Box::new(IntoIter {
- inner: self.iter(),
- key: Default::default(),
- value: Default::default(),
- })
- }
-}
-
-pub struct IntoIter<K, V>
-where
- K: Serialize + DeserializeOwned,
- V: Serialize + DeserializeOwned,
-{
- inner: sled::Iter,
- key: PhantomData<K>,
- value: PhantomData<V>,
-}
-
-impl<K, V> Iterator for IntoIter<K, V>
-where
- K: Serialize + DeserializeOwned,
- V: Serialize + DeserializeOwned,
-{
- type Item = (K, V);
-
- fn next(&mut self) -> Option<Self::Item> {
- self.inner.next().map(|res| {
- let (key_bytes, value_bytes) = res.expect("Failed to get next record from sled tree");
- (
- bincode::deserialize(&key_bytes).expect("Failed to deserialize key"),
- bincode::deserialize(&value_bytes).expect("Failed to deserialize value"),
- )
- })
- }
-}
diff --git a/src/lib.rs b/src/lib.rs
index 0507e230..c33bca29 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -82,7 +82,6 @@ pub struct WebgraphLocalConfig {
#[derive(Debug, Serialize, Deserialize, Clone)]
#[serde(tag = "type")]
pub enum WarcSource {
- S3(S3Config),
HTTP(HttpConfig),
Local(LocalConfig),
}
@@ -91,12 +90,6 @@ impl WarcSource {
pub fn paths(&self) -> Result<Vec<String>> {
let mut warc_paths = Vec::new();
match &self {
- WarcSource::S3(config) => {
- let file = File::open(&config.warc_paths_file)?;
- for line in io::BufReader::new(file).lines() {
- warc_paths.push(line?);
- }
- }
WarcSource::HTTP(config) => {
let file = File::open(&config.warc_paths_file)?;
for line in io::BufReader::new(file).lines() {
@@ -118,14 +111,6 @@ pub struct LocalConfig {
names: Vec<String>,
}
-#[derive(Debug, Serialize, Deserialize, Clone)]
-pub struct S3Config {
- name: String,
- endpoint: String,
- warc_paths_file: String,
- bucket: String,
-}
-
#[derive(Debug, Serialize, Deserialize, Clone)]
pub struct HttpConfig {
base_url: String,
@@ -134,15 +119,9 @@ pub struct HttpConfig {
#[derive(Error, Debug)]
pub enum Error {
- #[error("Failed to download object from S3")]
- S3DownloadError,
-
#[error("Failed to download object from HTTP")]
HTTPDownloadError(#[from] reqwest::Error),
- #[error("Failed to get the object from S3")]
- GetObjectError(#[from] rusoto_core::RusotoError<rusoto_s3::GetObjectError>),
-
#[error("Got an IO error")]
IOError(#[from] io::Error),
diff --git a/src/ranking/centrality_store.rs b/src/ranking/centrality_store.rs
index 59fc675b..5b89898f 100644
--- a/src/ranking/centrality_store.rs
+++ b/src/ranking/centrality_store.rs
@@ -16,42 +16,25 @@
use std::path::Path;
+use crate::kv::{rocksdb_store::RocksDbStore, Kv};
+
pub struct CentralityStore {
- inner: sled::Tree,
+ inner: Box<dyn Kv<String, f64>>,
}
impl CentralityStore {
pub fn new<P: AsRef<Path>>(path: P) -> Self {
- let db = sled::Config::default()
- .path(path)
- .use_compression(true)
- .mode(sled::Mode::LowSpace)
- .open()
- .expect("Failed to open database");
-
Self {
- inner: db
- .open_tree("centrality_store")
- .expect("Failed to open tree"),
+ inner: RocksDbStore::open(path),
}
}
pub fn insert(&mut self, key: String, centrality: f64) {
- let key_bytes = bincode::serialize(&key).expect("Failed to serialize key");
- let centrality_bytes =
- bincode::serialize(&centrality).expect("Failed to serialize centrality");
-
- self.inner
- .insert(key_bytes, centrality_bytes)
- .expect("Failed to insert into tree");
+ self.inner.insert(key, centrality);
}
pub fn get(&self, key: &str) -> Option<f64> {
- let key_bytes = bincode::serialize(key).expect("Failed to serialize key");
- self.inner
- .get(key_bytes)
- .expect("Failed to retrieve value from key")
- .map(|bytes| bincode::deserialize(&bytes).expect("Failed to deserialize value"))
+ self.inner.get(&key.to_string())
}
pub fn append(&mut self, it: impl Iterator<Item = (String, f64)>) {
@@ -59,6 +42,7 @@ impl CentralityStore {
.for_each(|(key, value)| {
self.insert(key, value);
});
- self.inner.flush().unwrap();
+
+ self.inner.flush();
}
}
diff --git a/src/warc.rs b/src/warc.rs
index 32d48930..9e174bc6 100644
--- a/src/warc.rs
+++ b/src/warc.rs
@@ -71,12 +71,6 @@ impl WarcFile {
pub(crate) fn download(source: WarcSource, path: &str) -> Result<Self> {
for dur in ExponentialBackoff::from_millis(10).take(5) {
let res = match source.clone() {
- WarcSource::S3(config) => WarcFile::download_from_s3(
- path,
- config.name.clone(),
- config.endpoint.clone(),
- config.bucket,
- ),
WarcSource::HTTP(config) => WarcFile::download_from_http(path, config.base_url),
WarcSource::Local(config) => WarcFile::load_from_folder(path, &config.folder),
};
@@ -122,39 +116,6 @@ impl WarcFile {
Ok(WarcFile::new(bytes))
}
-
- fn download_from_s3(
- _path: &str,
- _region_name: String,
- _region_endpoint: String,
- _bucket: String,
- ) -> Result<Self> {
- todo!();
- // let path = path.to_string();
- // let region = Region::Custom {
- // name: region_name,
- // endpoint: region_endpoint,
- // };
-
- // let client = S3Client::new(region);
-
- // let obj = client
- // .get_object(GetObjectRequest {
- // bucket,
- // key: path,
- // ..Default::default()
- // })
- // .await?;
-
- // let mut bytes = Vec::new();
- // obj.body
- // .ok_or(Error::S3DownloadError)?
- // .into_async_read()
- // .read_to_end(&mut bytes)
- // .await?;
-
- // Ok(WarcFile::new(bytes))
- }
}
#[derive(Debug)]
diff --git a/src/webgraph/graph_store.rs b/src/webgraph/graph_store.rs
index e9c483c6..1f4b6ad1 100644
--- a/src/webgraph/graph_store.rs
+++ b/src/webgraph/graph_store.rs
@@ -22,7 +22,7 @@ use lru::LruCache;
use serde::{de::DeserializeOwned, Serialize};
use super::{Edge, EdgeIterator, Node, NodeID, Store, StoredEdge};
-use crate::kv::{rocksdb_store::RocksDbStore, sled_store::SledStore, Kv};
+use crate::kv::{rocksdb_store::RocksDbStore, Kv};
pub(crate) struct Adjacency {
pub(crate) tree: BlockedCachedTree<NodeID, Vec<StoredEdge>>,
}
@@ -314,47 +314,6 @@ impl Store for RocksDbStore {
}
}
-impl Store for SledStore {
- fn open<P: AsRef<std::path::Path>>(path: P) -> GraphStore<Self> {
- let db = sled::Config::default()
- .path(path)
- .use_compression(true)
- .mode(sled::Mode::LowSpace)
- .open()
- .expect("Failed to open database");
-
- let adjacency = Box::new(db.open_tree("adjacency").expect("unable to open sled tree"));
- let reversed_adjacency = Box::new(
- db.open_tree("reversed_adjacency")
- .expect("unable to open sled tree"),
- );
- let node2id = Box::new(db.open_tree("node2id").expect("unable to open sled tree"));
- let id2node = Box::new(db.open_tree("id2node").expect("unable to open sled tree"));
- let meta = Box::new(db.open_tree("meta").expect("unable to open sled tree"));
-
- GraphStore {
- adjacency: RefCell::new(Adjacency {
- tree: BlockedCachedTree {
- inner: CachedTree::new(adjacency, 10_000),
- block_size: 1_024,
- },
- }),
- reversed_adjacency: RefCell::new(Adjacency {
- tree: BlockedCachedTree {
- inner: CachedTree::new(reversed_adjacency, 10_000),
- block_size: 1_024,
- },
- }),
- node2id: RefCell::new(CachedTree::new(node2id, 100_000)),
- id2node: RefCell::new(BlockedCachedTree {
- inner: CachedTree::new(id2node, 100_000),
- block_size: 1_024,
- }),
- meta: RefCell::new(CachedTree::new(meta, 1_000)),
- store: Default::default(),
- }
- }
-}
#[cfg(test)]
mod test {
use super::*;
diff --git a/src/webpage/just_text.rs b/src/webpage/just_text.rs
index b12f8545..9e479ad1 100644
--- a/src/webpage/just_text.rs
+++ b/src/webpage/just_text.rs
@@ -16,7 +16,6 @@
use std::collections::{HashMap, HashSet};
-use lazy_static::lazy_static;
use logos::{Lexer, Logos};
use whatlang::Lang;
@@ -70,8 +69,8 @@ macro_rules! include_stopwords {
}};
}
-lazy_static! {
- static ref STOPWORDS: HashMap<Lang, HashSet<String>> = {
+static STOPWORDS: once_cell::sync::Lazy<HashMap<Lang, HashSet<String>>> =
+ once_cell::sync::Lazy::new(|| {
include_stopwords!(
"../../stopwords/Afrikaans.txt" => Lang::Afr,
"../../stopwords/Arabic.txt" => Lang::Ara,
@@ -120,8 +119,7 @@ lazy_static! {
"../../stopwords/Spanish.txt" => Lang::Spa,
"../../stopwords/Japanese.txt" => Lang::Jpn
)
- };
-}
+ });
pub struct JustText {
pub max_link_density: f64,