Skip to content

web search done right

leechy ranking annotation experiment

Mikkel Denkerf3315c4b422b
diff --git a/Cargo.lock b/Cargo.lock
index 4aa7859e..60b6d1de 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -103,9 +103,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
[[package]]
name = "anstream"
-version = "0.6.14"
+version = "0.6.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b"
+checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"
dependencies = [
"anstyle",
"anstyle-parse",
@@ -118,33 +118,33 @@ dependencies = [
[[package]]
name = "anstyle"
-version = "1.0.7"
+version = "1.0.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"
+checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"
[[package]]
name = "anstyle-parse"
-version = "0.2.4"
+version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"
+checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"
dependencies = [
"utf8parse",
]
[[package]]
name = "anstyle-query"
-version = "1.1.0"
+version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391"
+checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"
dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "anstyle-wincon"
-version = "3.0.3"
+version = "3.0.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19"
+checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"
dependencies = [
"anstyle",
"windows-sys 0.52.0",
@@ -598,9 +598,9 @@ checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"
[[package]]
name = "bytemuck"
-version = "1.16.1"
+version = "1.16.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e"
+checksum = "102087e286b4677862ea56cf8fc58bb2cdfa8725c40ffb80fe3a008eb7f2fc83"
dependencies = [
"bytemuck_derive",
]
@@ -636,9 +636,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"
[[package]]
name = "bytes"
-version = "1.6.1"
+version = "1.7.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952"
+checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50"
[[package]]
name = "bzip2"
@@ -746,9 +746,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"
[[package]]
name = "cc"
-version = "1.1.6"
+version = "1.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f"
+checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549"
dependencies = [
"jobserver",
"libc",
@@ -857,9 +857,9 @@ dependencies = [
[[package]]
name = "clap"
-version = "4.5.10"
+version = "4.5.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8f6b81fb3c84f5563d509c59b5a48d935f689e993afa90fe39047f05adef9142"
+checksum = "c937d4061031a6d0c8da4b9a4f98a172fc2976dfb1c19213a9cf7d0d3c837e36"
dependencies = [
"clap_builder",
"clap_derive",
@@ -867,9 +867,9 @@ dependencies = [
[[package]]
name = "clap_builder"
-version = "4.5.10"
+version = "4.5.14"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ca6706fd5224857d9ac5eb9355f6683563cc0541c7cd9d014043b57cbec78ac"
+checksum = "85379ba512b21a328adf887e85f7742d12e96eb31f3ef077df4ffc26b506ffed"
dependencies = [
"anstream",
"anstyle",
@@ -879,9 +879,9 @@ dependencies = [
[[package]]
name = "clap_derive"
-version = "4.5.8"
+version = "4.5.13"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085"
+checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"
dependencies = [
"heck 0.5.0",
"proc-macro2",
@@ -891,9 +891,9 @@ dependencies = [
[[package]]
name = "clap_lex"
-version = "0.7.1"
+version = "0.7.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70"
+checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"
[[package]]
name = "client-wasm"
@@ -923,9 +923,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"
[[package]]
name = "colorchoice"
-version = "1.0.1"
+version = "1.0.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"
+checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"
[[package]]
name = "console"
@@ -1609,14 +1609,14 @@ dependencies = [
[[package]]
name = "filetime"
-version = "0.2.23"
+version = "0.2.24"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd"
+checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550"
dependencies = [
"cfg-if",
"libc",
- "redox_syscall 0.4.1",
- "windows-sys 0.52.0",
+ "libredox",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -1627,9 +1627,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"
[[package]]
name = "flate2"
-version = "1.0.30"
+version = "1.0.31"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae"
+checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920"
dependencies = [
"crc32fast",
"miniz_oxide",
@@ -1990,7 +1990,7 @@ dependencies = [
"futures-sink",
"futures-util",
"http 0.2.12",
- "indexmap 2.2.6",
+ "indexmap 2.3.0",
"slab",
"tokio",
"tokio-util",
@@ -2279,9 +2279,9 @@ dependencies = [
[[package]]
name = "hyper-util"
-version = "0.1.6"
+version = "0.1.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956"
+checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9"
dependencies = [
"bytes",
"futures-channel",
@@ -2397,9 +2397,9 @@ dependencies = [
[[package]]
name = "indexmap"
-version = "2.2.6"
+version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"
+checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0"
dependencies = [
"equivalent",
"hashbrown 0.14.5",
@@ -2507,9 +2507,9 @@ dependencies = [
[[package]]
name = "is_terminal_polyfill"
-version = "1.70.0"
+version = "1.70.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"
+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
[[package]]
name = "itertools"
@@ -2651,6 +2651,18 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"
+[[package]]
+name = "leechy"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "kuchiki",
+ "rand",
+ "reqwest 0.11.27",
+ "serde",
+ "toml",
+]
+
[[package]]
name = "lending-iter"
version = "0.1.0"
@@ -2692,6 +2704,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"
dependencies = [
"bitflags 2.6.0",
"libc",
+ "redox_syscall",
]
[[package]]
@@ -2765,9 +2778,9 @@ dependencies = [
[[package]]
name = "lru"
-version = "0.12.3"
+version = "0.12.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc"
+checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904"
dependencies = [
"hashbrown 0.14.5",
]
@@ -2871,7 +2884,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519"
dependencies = [
"cfg-if",
- "rayon",
]
[[package]]
@@ -3157,11 +3169,11 @@ dependencies = [
[[package]]
name = "num_enum"
-version = "0.7.2"
+version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845"
+checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179"
dependencies = [
- "num_enum_derive 0.7.2",
+ "num_enum_derive 0.7.3",
]
[[package]]
@@ -3178,9 +3190,9 @@ dependencies = [
[[package]]
name = "num_enum_derive"
-version = "0.7.2"
+version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b"
+checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56"
dependencies = [
"proc-macro-crate 3.1.0",
"proc-macro2",
@@ -3202,9 +3214,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
[[package]]
name = "object"
-version = "0.36.2"
+version = "0.36.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e"
+checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9"
dependencies = [
"memchr",
]
@@ -3403,7 +3415,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
dependencies = [
"cfg-if",
"libc",
- "redox_syscall 0.5.3",
+ "redox_syscall",
"smallvec",
"windows-targets 0.52.6",
]
@@ -3475,7 +3487,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"
dependencies = [
"fixedbitset",
- "indexmap 2.2.6",
+ "indexmap 2.3.0",
]
[[package]]
@@ -3682,9 +3694,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"
[[package]]
name = "ppv-lite86"
-version = "0.2.17"
+version = "0.2.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
+checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
+dependencies = [
+ "zerocopy",
+]
[[package]]
name = "precomputed-hash"
@@ -3865,16 +3880,17 @@ dependencies = [
[[package]]
name = "quinn"
-version = "0.11.2"
+version = "0.11.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad"
+checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156"
dependencies = [
"bytes",
"pin-project-lite",
"quinn-proto",
"quinn-udp",
- "rustc-hash",
+ "rustc-hash 2.0.0",
"rustls",
+ "socket2",
"thiserror",
"tokio",
"tracing",
@@ -3882,14 +3898,14 @@ dependencies = [
[[package]]
name = "quinn-proto"
-version = "0.11.3"
+version = "0.11.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe"
+checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd"
dependencies = [
"bytes",
"rand",
"ring",
- "rustc-hash",
+ "rustc-hash 2.0.0",
"rustls",
"slab",
"thiserror",
@@ -3906,6 +3922,7 @@ dependencies = [
"libc",
"once_cell",
"socket2",
+ "tracing",
"windows-sys 0.52.0",
]
@@ -4019,16 +4036,15 @@ dependencies = [
[[package]]
name = "ravif"
-version = "0.11.8"
+version = "0.11.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c6ba61c28ba24c0cf8406e025cb29a742637e3f70776e61c27a8a8b72a042d12"
+checksum = "a8f0bfd976333248de2078d350bfdf182ff96e168a24d23d2436cef320dd4bdd"
dependencies = [
"avif-serialize",
"imgref",
"loop9",
"quick-error 2.0.1",
"rav1e",
- "rayon",
"rgb",
]
@@ -4087,15 +4103,6 @@ dependencies = [
"libc",
]
-[[package]]
-name = "redox_syscall"
-version = "0.4.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"
-dependencies = [
- "bitflags 1.3.2",
-]
-
[[package]]
name = "redox_syscall"
version = "0.5.3"
@@ -4118,9 +4125,9 @@ dependencies = [
[[package]]
name = "regex"
-version = "1.10.5"
+version = "1.10.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f"
+checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"
dependencies = [
"aho-corasick 1.1.3",
"memchr",
@@ -4234,7 +4241,7 @@ dependencies = [
"pin-project-lite",
"quinn",
"rustls",
- "rustls-pemfile 2.1.2",
+ "rustls-pemfile 2.1.3",
"rustls-pki-types",
"serde",
"serde_json",
@@ -4253,9 +4260,9 @@ dependencies = [
[[package]]
name = "rgb"
-version = "0.8.45"
+version = "0.8.48"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ade4539f42266ded9e755c605bdddf546242b2c961b03b06a7375260788a0523"
+checksum = "0f86ae463694029097b846d8f99fd5536740602ae00022c0c50c5600720b2f71"
dependencies = [
"bytemuck",
]
@@ -4415,6 +4422,12 @@ version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
+[[package]]
+name = "rustc-hash"
+version = "2.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152"
+
[[package]]
name = "rustc_version"
version = "0.4.0"
@@ -4462,9 +4475,9 @@ dependencies = [
[[package]]
name = "rustls-pemfile"
-version = "2.1.2"
+version = "2.1.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d"
+checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425"
dependencies = [
"base64 0.22.1",
"rustls-pki-types",
@@ -4472,9 +4485,9 @@ dependencies = [
[[package]]
name = "rustls-pki-types"
-version = "1.7.0"
+version = "1.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d"
+checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0"
[[package]]
name = "rustls-webpki"
@@ -4536,9 +4549,9 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
[[package]]
name = "safetensors"
-version = "0.4.3"
+version = "0.4.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ced76b22c7fba1162f11a5a75d9d8405264b467a07ae0c9c29be119b9297db9"
+checksum = "7725d4d98fa515472f43a6e2bbf956c48e06b89bb50593a040e5945160214450"
dependencies = [
"serde",
"serde_json",
@@ -4687,9 +4700,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"
[[package]]
name = "serde"
-version = "1.0.204"
+version = "1.0.205"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"
+checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150"
dependencies = [
"serde_derive",
]
@@ -4707,9 +4720,9 @@ dependencies = [
[[package]]
name = "serde_derive"
-version = "1.0.204"
+version = "1.0.205"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"
+checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1"
dependencies = [
"proc-macro2",
"quote",
@@ -4718,11 +4731,12 @@ dependencies = [
[[package]]
name = "serde_json"
-version = "1.0.120"
+version = "1.0.122"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"
+checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
dependencies = [
"itoa",
+ "memchr",
"ryu",
"serde",
]
@@ -4748,9 +4762,9 @@ dependencies = [
[[package]]
name = "serde_spanned"
-version = "0.6.6"
+version = "0.6.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0"
+checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d"
dependencies = [
"serde",
]
@@ -5008,7 +5022,7 @@ dependencies = [
"robotstxt",
"rust-s3",
"rust-stemmers",
- "rustc-hash",
+ "rustc-hash 1.1.0",
"scylla",
"serde",
"serde_json",
@@ -5270,7 +5284,7 @@ dependencies = [
"rayon",
"regex",
"rust-stemmers",
- "rustc-hash",
+ "rustc-hash 1.1.0",
"serde",
"serde_json",
"smallvec",
@@ -5302,20 +5316,21 @@ dependencies = [
[[package]]
name = "target-lexicon"
-version = "0.12.15"
+version = "0.12.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
[[package]]
name = "tempfile"
-version = "3.10.1"
+version = "3.12.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"
+checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
dependencies = [
"cfg-if",
"fastrand",
+ "once_cell",
"rustix",
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -5504,9 +5519,9 @@ dependencies = [
[[package]]
name = "tokio"
-version = "1.39.1"
+version = "1.39.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d040ac2b29ab03b09d4129c2f5bbd012a3ac2f79d38ff506a4bf8dd34b0eac8a"
+checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1"
dependencies = [
"backtrace",
"bytes",
@@ -5579,21 +5594,21 @@ dependencies = [
[[package]]
name = "toml"
-version = "0.8.15"
+version = "0.8.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28"
+checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
dependencies = [
"serde",
"serde_spanned",
"toml_datetime",
- "toml_edit 0.22.16",
+ "toml_edit 0.22.20",
]
[[package]]
name = "toml_datetime"
-version = "0.6.6"
+version = "0.6.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf"
+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
dependencies = [
"serde",
]
@@ -5604,7 +5619,7 @@ version = "0.19.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"
dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.3.0",
"toml_datetime",
"winnow 0.5.40",
]
@@ -5615,22 +5630,22 @@ version = "0.21.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"
dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.3.0",
"toml_datetime",
"winnow 0.5.40",
]
[[package]]
name = "toml_edit"
-version = "0.22.16"
+version = "0.22.20"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788"
+checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"
dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.3.0",
"serde",
"serde_spanned",
"toml_datetime",
- "winnow 0.6.15",
+ "winnow 0.6.18",
]
[[package]]
@@ -5920,7 +5935,7 @@ version = "4.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5afb1a60e207dca502682537fefcfd9921e71d0b83e9576060f09abc6efab23"
dependencies = [
- "indexmap 2.2.6",
+ "indexmap 2.3.0",
"serde",
"serde_json",
"utoipa-gen",
@@ -6007,9 +6022,9 @@ checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b"
[[package]]
name = "version_check"
-version = "0.9.4"
+version = "0.9.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
[[package]]
name = "virtue"
@@ -6193,11 +6208,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
[[package]]
name = "winapi-util"
-version = "0.1.8"
+version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b"
+checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"
dependencies = [
- "windows-sys 0.52.0",
+ "windows-sys 0.59.0",
]
[[package]]
@@ -6233,6 +6248,15 @@ dependencies = [
"windows-targets 0.52.6",
]
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
[[package]]
name = "windows-targets"
version = "0.48.5"
@@ -6365,9 +6389,9 @@ dependencies = [
[[package]]
name = "winnow"
-version = "0.6.15"
+version = "0.6.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "557404e450152cd6795bb558bca69e43c585055f4606e3bcae5894fc6dac9ba0"
+checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
dependencies = [
"memchr",
]
@@ -6414,9 +6438,9 @@ dependencies = [
[[package]]
name = "xxhash-rust"
-version = "0.8.11"
+version = "0.8.12"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "63658493314859b4dfdf3fb8c1defd61587839def09582db50b8a4e93afca6bb"
+checksum = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984"
[[package]]
name = "yansi"
@@ -6463,6 +6487,7 @@ version = "0.7.35"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
dependencies = [
+ "byteorder 1.5.0",
"zerocopy-derive",
]
@@ -6545,8 +6570,8 @@ dependencies = [
"crossbeam-utils",
"displaydoc",
"flate2",
- "indexmap 2.2.6",
- "num_enum 0.7.2",
+ "indexmap 2.3.0",
+ "num_enum 0.7.3",
"thiserror",
]
@@ -6574,7 +6599,7 @@ version = "0.13.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9"
dependencies = [
- "zstd-safe 7.2.0",
+ "zstd-safe 7.2.1",
]
[[package]]
@@ -6589,18 +6614,18 @@ dependencies = [
[[package]]
name = "zstd-safe"
-version = "7.2.0"
+version = "7.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa"
+checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059"
dependencies = [
"zstd-sys",
]
[[package]]
name = "zstd-sys"
-version = "2.0.12+zstd.1.5.6"
+version = "2.0.13+zstd.1.5.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13"
+checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"
dependencies = [
"cc",
"pkg-config",
diff --git a/Cargo.toml b/Cargo.toml
index 9373d7dc..423bb260 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,6 +13,7 @@ members = [
"crates/lending-iter",
"crates/robotstxt",
"crates/tantivy",
+ "crates/leechy",
"fuzz",
]
resolver = "2"
diff --git a/crates/kuchiki/src/iter.rs b/crates/kuchiki/src/iter.rs
index fa838386..02405e8a 100644
--- a/crates/kuchiki/src/iter.rs
+++ b/crates/kuchiki/src/iter.rs
@@ -4,6 +4,8 @@ use std::borrow::Borrow;
use std::cell::RefCell;
use std::iter::Rev;
+use selectors::Element;
+
use crate::node_data_ref::NodeDataRef;
use crate::select::Selectors;
use crate::tree::{ElementData, NodeRef};
@@ -170,6 +172,12 @@ impl NodeRef {
self.inclusive_descendants().select(selectors)
}
+ /// Return the node that matches the given xpath expression.
+ #[inline]
+ pub fn select_xpath(&self, xpath: &str) -> XPathSelect<Elements<Descendants>> {
+ XPathSelect::new(self.inclusive_descendants().elements(), xpath)
+ }
+
/// Return the first inclusive descendants element that match the given selector list.
#[inline]
#[must_use]
@@ -179,6 +187,69 @@ impl NodeRef {
}
}
+/// An iterator of nodes that match a given xpath selector.
+pub struct XPathSelect<I> {
+ iter: I,
+ xpath: Vec<crate::xpath::Expr>,
+}
+
+impl<I> XPathSelect<I> {
+ /// Create a new `XPathSelect` iterator.
+ pub fn new(iter: I, xpath: &str) -> Self {
+ let xpath = crate::xpath::Expr::parse(xpath);
+ Self { iter, xpath }
+ }
+}
+
+impl<I> Iterator for XPathSelect<I>
+where
+ I: Iterator<Item = NodeDataRef<ElementData>>,
+{
+ type Item = NodeDataRef<ElementData>;
+
+ fn next(&mut self) -> Option<Self::Item> {
+ self.iter.by_ref().find(|elem| {
+ let mut elem = Some(elem.clone());
+ let mut expr_index = self.xpath.len();
+
+ while expr_index > 0 {
+ let expr = &self.xpath[expr_index - 1];
+
+ elem = match elem {
+ Some(elem) => {
+ if !expr.matches(&elem) {
+ return false;
+ }
+
+ elem.parent_element()
+ }
+
+ None => {
+ return matches!(
+ expr,
+ crate::xpath::Expr::Root | crate::xpath::Expr::Wildcard
+ );
+ }
+ };
+
+ if matches!(expr, crate::xpath::Expr::Wildcard)
+ && expr_index > 1
+ && elem
+ .as_ref()
+ .map(|elem| !self.xpath[expr_index - 2].matches(elem))
+ .unwrap_or(false)
+ {
+ expr_index -= 0;
+ } else {
+ expr_index -= 1;
+ }
+ }
+
+ true
+ })
+ }
+}
+
#[derive(Debug, Clone)]
struct State<T> {
next: T,
diff --git a/crates/kuchiki/src/lib.rs b/crates/kuchiki/src/lib.rs
index bbe14e5f..897d0049 100644
--- a/crates/kuchiki/src/lib.rs
+++ b/crates/kuchiki/src/lib.rs
@@ -18,6 +18,7 @@ mod node_data_ref;
mod parser;
mod select;
mod serializer;
+mod xpath;
#[cfg(test)]
mod tests;
diff --git a/crates/kuchiki/src/tests.rs b/crates/kuchiki/src/tests.rs
index 83edbb6c..214320d6 100644
--- a/crates/kuchiki/src/tests.rs
+++ b/crates/kuchiki/src/tests.rs
@@ -189,3 +189,83 @@ fn specificity() {
assert!(specificities[0] > specificities[2]);
assert!(specificities[1] > specificities[2]);
}
+
+#[test]
+fn select_xpath() {
+ let html = r"
+<title>Test case</title>
+<p class=foo>Foo</p>
+<p>Bar</p>
+<p class=foo>Foo</p>
+";
+
+ let document = parse_html().one(html);
+ let nodes: Vec<_> = document.select_xpath("/html/body/p").collect();
+
+ assert_eq!(nodes.len(), 3);
+
+ let node = &nodes[0];
+ assert_eq!(node.name.local.as_ref(), "p");
+ assert_eq!(node.attributes.borrow().get("class"), Some("foo"));
+
+ let node = &nodes[1];
+ assert_eq!(node.name.local.as_ref(), "p");
+ assert_eq!(node.attributes.borrow().get("class"), None);
+
+ let node = &nodes[2];
+ assert_eq!(node.name.local.as_ref(), "p");
+ assert_eq!(node.attributes.borrow().get("class"), Some("foo"));
+
+ let nodes: Vec<_> = document.select_xpath("/html/body/p[2]").collect();
+
+ assert_eq!(nodes.len(), 1);
+
+ let node = &nodes[0];
+ assert_eq!(node.name.local.as_ref(), "p");
+ assert_eq!(node.attributes.borrow().get("class"), None);
+
+ let nodes: Vec<_> = document.select_xpath("/html/head/title").collect();
+ assert_eq!(nodes.len(), 1);
+}
+
+#[test]
+fn xpath_wildcard() {
+ let html = r"
+<title>Test case</title>
+";
+
+ let document = parse_html().one(html);
+
+ let nodes: Vec<_> = document.select_xpath("/title").collect();
+ assert_eq!(nodes.len(), 0);
+
+ let nodes: Vec<_> = document.select_xpath("./title").collect();
+ assert_eq!(nodes.len(), 1);
+
+ let nodes: Vec<_> = document.select_xpath("/./title").collect();
+ assert_eq!(nodes.len(), 1);
+}
+
+#[test]
+fn xpath_contains() {
+ let html = r#"
+<p class="foo bar">Test case</p>
+"#;
+
+ let document = parse_html().one(html);
+
+ let nodes: Vec<_> = document
+ .select_xpath("./p[contains(@class, 'foo')]")
+ .collect();
+ assert_eq!(nodes.len(), 1);
+
+ let nodes: Vec<_> = document
+ .select_xpath("./p[contains(@class, 'bar')]")
+ .collect();
+ assert_eq!(nodes.len(), 1);
+
+ let nodes: Vec<_> = document
+ .select_xpath("./p[contains(@class, 'baz')]")
+ .collect();
+ assert_eq!(nodes.len(), 0);
+}
diff --git a/crates/kuchiki/src/xpath.rs b/crates/kuchiki/src/xpath.rs
new file mode 100644
index 00000000..227ba3c5
--- /dev/null
+++ b/crates/kuchiki/src/xpath.rs
@@ -0,0 +1,297 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use selectors::Element;
+
+use crate::{iter::NodeIterator, ElementData, NodeDataRef};
+
+#[derive(Debug, PartialEq)]
+pub enum Predicate {
+ Nth(usize),
+ Contains { key: ContainsKey, value: String },
+}
+
+#[derive(Debug, PartialEq)]
+pub enum ContainsKey {
+ Text,
+ Attr(String),
+}
+
+impl Predicate {
+ pub fn parse(s: &str) -> Option<Self> {
+ if let Ok(num) = s.parse::<usize>() {
+ if num > 0 {
+ return Some(Predicate::Nth(num - 1));
+ }
+ }
+
+ if s.starts_with("contains(text(),") {
+ let value = s
+ .trim_start_matches("contains(text(),")
+ .trim_end_matches(')');
+ return Some(Predicate::Contains {
+ key: ContainsKey::Text,
+ value: value.to_string(),
+ });
+ }
+
+ if s.starts_with("contains(@") {
+ let attr_val = s.trim_start_matches("contains(@").trim_end_matches(')');
+
+ let mut parts = attr_val.split(',');
+
+ if let (Some(attr), Some(value)) = (parts.next(), parts.next()) {
+ return Some(Predicate::Contains {
+ key: ContainsKey::Attr(attr.to_string()),
+ value: value
+ .trim()
+ .trim_start_matches(|c| matches!(c, '\'' | '"'))
+ .trim_end_matches(|c| matches!(c, '\'' | '"'))
+ .to_string(),
+ });
+ }
+ }
+
+ None
+ }
+
+ pub fn matches(&self, elem: &NodeDataRef<ElementData>) -> bool {
+ match self {
+ Predicate::Nth(nth) => {
+ let parent = elem.parent_element().unwrap();
+ parent
+ .as_node()
+ .children()
+ .elements()
+ .nth(*nth)
+ .map(|child| &child == elem)
+ .unwrap_or(false)
+ }
+
+ Predicate::Contains { key, value } => match key {
+ ContainsKey::Text => elem.text_contents().contains(value),
+ ContainsKey::Attr(attr) => elem
+ .attributes
+ .borrow()
+ .get(attr.as_str())
+ .map_or(false, |v| v.split_ascii_whitespace().any(|v| v == value)),
+ },
+ }
+ }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum Expr {
+ Root,
+ Wildcard,
+ Child {
+ name: String,
+ pred: Option<Predicate>,
+ },
+}
+
+impl Expr {
+ pub fn parse(s: &str) -> Vec<Self> {
+ let mut tokens = Vec::new();
+ let mut current = String::new();
+ let mut in_brackets = false;
+
+ for c in s.chars() {
+ match c {
+ '/' if !in_brackets => {
+ if !current.is_empty() {
+ tokens.push(current);
+ current = String::new();
+ }
+ tokens.push("/".to_string());
+ }
+ '[' => {
+ in_brackets = true;
+ tokens.push(current);
+ current = String::new();
+ current.push('[');
+ }
+ ']' => {
+ in_brackets = false;
+ current.push(']');
+ tokens.push(current);
+ current = String::new();
+ }
+ _ => current.push(c),
+ }
+ }
+
+ if !current.is_empty() {
+ tokens.push(current);
+ }
+
+ let mut result = Vec::new();
+ let mut is_root = true;
+
+ let mut i = 0;
+ while i < tokens.len() {
+ match tokens[i].as_str() {
+ "/" => {
+ if is_root {
+ result.push(Expr::Root);
+ is_root = false;
+ }
+ }
+ _ => {
+ let name = tokens[i].to_string();
+
+ if name == "." || name == "*" {
+ result.push(Expr::Wildcard);
+ i += 1;
+ is_root = false;
+ continue;
+ }
+
+ let mut pred = None;
+
+ if i + 1 < tokens.len()
+ && tokens[i + 1].starts_with('[')
+ && tokens[i + 1].ends_with(']')
+ {
+ if let Some(p) =
+ Predicate::parse(&tokens[i + 1][1..tokens[i + 1].len() - 1])
+ {
+ pred = Some(p);
+ i += 1;
+ }
+ }
+
+ result.push(Expr::Child {
+ name: name.trim_start_matches('/').to_string(),
+ pred,
+ });
+ }
+ }
+ i += 1;
+ }
+
+ result
+ }
+
+ pub fn matches(&self, elem: &NodeDataRef<ElementData>) -> bool {
+ match self {
+ Expr::Root => elem.parent_element().is_none(),
+ Expr::Child { name, pred } => {
+ if elem.name.local.as_ref() != name {
+ return false;
+ }
+
+ if let Some(pred) = pred {
+ pred.matches(elem)
+ } else {
+ true
+ }
+ }
+ Expr::Wildcard => true,
+ }
+ }
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_expr_parse() {
+ assert_eq!(Expr::parse("/"), vec![Expr::Root]);
+ assert_eq!(
+ Expr::parse("/html"),
+ vec![
+ Expr::Root,
+ Expr::Child {
+ name: "html".to_string(),
+ pred: None
+ }
+ ]
+ );
+ assert_eq!(
+ Expr::parse("/html[1]"),
+ vec![
+ Expr::Root,
+ Expr::Child {
+ name: "html".to_string(),
+ pred: Some(Predicate::Nth(0))
+ }
+ ]
+ );
+ assert_eq!(
+ Expr::parse("/html[1]/body"),
+ vec![
+ Expr::Root,
+ Expr::Child {
+ name: "html".to_string(),
+ pred: Some(Predicate::Nth(0))
+ },
+ Expr::Child {
+ name: "body".to_string(),
+ pred: None
+ }
+ ]
+ );
+ assert_eq!(
+ Expr::parse("/html[1]/body[1]"),
+ vec![
+ Expr::Root,
+ Expr::Child {
+ name: "html".to_string(),
+ pred: Some(Predicate::Nth(0))
+ },
+ Expr::Child {
+ name: "body".to_string(),
+ pred: Some(Predicate::Nth(0))
+ }
+ ]
+ );
+ assert_eq!(
+ Expr::parse("/html[1]/body[1]"),
+ vec![
+ Expr::Root,
+ Expr::Child {
+ name: "html".to_string(),
+ pred: Some(Predicate::Nth(0))
+ },
+ Expr::Child {
+ name: "body".to_string(),
+ pred: Some(Predicate::Nth(0))
+ },
+ ]
+ );
+
+ assert_eq!(
+ Expr::parse("/html/body/div[4]"),
+ vec![
+ Expr::Root,
+ Expr::Child {
+ name: "html".to_string(),
+ pred: None
+ },
+ Expr::Child {
+ name: "body".to_string(),
+ pred: None
+ },
+ Expr::Child {
+ name: "div".to_string(),
+ pred: Some(Predicate::Nth(3))
+ }
+ ]
+ );
+ }
+}
diff --git a/crates/leechy-py/.gitignore b/crates/leechy-py/.gitignore
new file mode 100644
index 00000000..c8f04429
--- /dev/null
+++ b/crates/leechy-py/.gitignore
@@ -0,0 +1,72 @@
+/target
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+.pytest_cache/
+*.py[cod]
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+.venv/
+env/
+bin/
+build/
+develop-eggs/
+dist/
+eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+include/
+man/
+venv/
+*.egg-info/
+.installed.cfg
+*.egg
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+pip-selfcheck.json
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.coverage
+.cache
+nosetests.xml
+coverage.xml
+
+# Translations
+*.mo
+
+# Mr Developer
+.mr.developer.cfg
+.project
+.pydevproject
+
+# Rope
+.ropeproject
+
+# Django stuff:
+*.log
+*.pot
+
+.DS_Store
+
+# Sphinx documentation
+docs/_build/
+
+# PyCharm
+.idea/
+
+# VSCode
+.vscode/
+
+# Pyenv
+.python-version
diff --git a/crates/leechy-py/Cargo.lock b/crates/leechy-py/Cargo.lock
new file mode 100644
index 00000000..467ffc1d
--- /dev/null
+++ b/crates/leechy-py/Cargo.lock
@@ -0,0 +1,1823 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "addr2line"
+version = "0.22.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678"
+dependencies = [
+ "gimli",
+]
+
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
+[[package]]
+name = "anyhow"
+version = "1.0.86"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"
+dependencies = [
+ "backtrace",
+]
+
+[[package]]
+name = "autocfg"
+version = "1.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"
+
+[[package]]
+name = "backtrace"
+version = "0.3.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if",
+ "libc",
+ "miniz_oxide",
+ "object",
+ "rustc-demangle",
+]
+
+[[package]]
+name = "base64"
+version = "0.21.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "bitflags"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"
+
+[[package]]
+name = "bumpalo"
+version = "3.16.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"
+
+[[package]]
+name = "byteorder"
+version = "1.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
+
+[[package]]
+name = "bytes"
+version = "1.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50"
+
+[[package]]
+name = "cc"
+version = "1.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549"
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "core-foundation"
+version = "0.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "core-foundation-sys"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"
+
+[[package]]
+name = "cssparser"
+version = "0.31.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b3df4f93e5fbbe73ec01ec8d3f68bba73107993a5b1e7519273c32db9b0d5be"
+dependencies = [
+ "cssparser-macros",
+ "dtoa-short",
+ "itoa",
+ "phf 0.11.2",
+ "smallvec",
+]
+
+[[package]]
+name = "cssparser-macros"
+version = "0.6.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"
+dependencies = [
+ "quote",
+ "syn 2.0.72",
+]
+
+[[package]]
+name = "derive_more"
+version = "0.99.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.72",
+]
+
+[[package]]
+name = "dtoa"
+version = "1.0.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653"
+
+[[package]]
+name = "dtoa-short"
+version = "0.3.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"
+dependencies = [
+ "dtoa",
+]
+
+[[package]]
+name = "encoding_rs"
+version = "0.8.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "equivalent"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"
+
+[[package]]
+name = "errno"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba"
+dependencies = [
+ "libc",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "fastrand"
+version = "2.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"
+
+[[package]]
+name = "fnv"
+version = "1.0.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
+
+[[package]]
+name = "foreign-types"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"
+dependencies = [
+ "foreign-types-shared",
+]
+
+[[package]]
+name = "foreign-types-shared"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
+
+[[package]]
+name = "form_urlencoded"
+version = "1.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"
+dependencies = [
+ "percent-encoding",
+]
+
+[[package]]
+name = "futf"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"
+dependencies = [
+ "mac",
+ "new_debug_unreachable",
+]
+
+[[package]]
+name = "futures-channel"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"
+dependencies = [
+ "futures-core",
+]
+
+[[package]]
+name = "futures-core"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"
+
+[[package]]
+name = "futures-io"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.72",
+]
+
+[[package]]
+name = "futures-sink"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"
+
+[[package]]
+name = "futures-task"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"
+
+[[package]]
+name = "futures-util"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"
+dependencies = [
+ "futures-core",
+ "futures-io",
+ "futures-macro",
+ "futures-sink",
+ "futures-task",
+ "memchr",
+ "pin-project-lite",
+ "pin-utils",
+ "slab",
+]
+
+[[package]]
+name = "fxhash"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
+dependencies = [
+ "byteorder",
+]
+
+[[package]]
+name = "getrandom"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "wasi",
+]
+
+[[package]]
+name = "gimli"
+version = "0.29.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"
+
+[[package]]
+name = "h2"
+version = "0.3.26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8"
+dependencies = [
+ "bytes",
+ "fnv",
+ "futures-core",
+ "futures-sink",
+ "futures-util",
+ "http",
+ "indexmap 2.3.0",
+ "slab",
+ "tokio",
+ "tokio-util",
+ "tracing",
+]
+
+[[package]]
+name = "hashbrown"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
+
+[[package]]
+name = "hashbrown"
+version = "0.14.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
+
+[[package]]
+name = "heck"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
+
+[[package]]
+name = "hermit-abi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"
+
+[[package]]
+name = "html5ever"
+version = "0.26.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7"
+dependencies = [
+ "log",
+ "mac",
+ "markup5ever",
+ "proc-macro2",
+ "quote",
+ "syn 1.0.109",
+]
+
+[[package]]
+name = "http"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"
+dependencies = [
+ "bytes",
+ "fnv",
+ "itoa",
+]
+
+[[package]]
+name = "http-body"
+version = "0.4.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"
+dependencies = [
+ "bytes",
+ "http",
+ "pin-project-lite",
+]
+
+[[package]]
+name = "httparse"
+version = "1.9.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9"
+
+[[package]]
+name = "httpdate"
+version = "1.0.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
+
+[[package]]
+name = "hyper"
+version = "0.14.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9"
+dependencies = [
+ "bytes",
+ "futures-channel",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "httparse",
+ "httpdate",
+ "itoa",
+ "pin-project-lite",
+ "socket2",
+ "tokio",
+ "tower-service",
+ "tracing",
+ "want",
+]
+
+[[package]]
+name = "hyper-tls"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"
+dependencies = [
+ "bytes",
+ "hyper",
+ "native-tls",
+ "tokio",
+ "tokio-native-tls",
+]
+
+[[package]]
+name = "idna"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"
+dependencies = [
+ "unicode-bidi",
+ "unicode-normalization",
+]
+
+[[package]]
+name = "indexmap"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
+dependencies = [
+ "autocfg",
+ "hashbrown 0.12.3",
+]
+
+[[package]]
+name = "indexmap"
+version = "2.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0"
+dependencies = [
+ "equivalent",
+ "hashbrown 0.14.5",
+]
+
+[[package]]
+name = "indoc"
+version = "2.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5"
+
+[[package]]
+name = "ipnet"
+version = "2.9.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"
+
+[[package]]
+name = "itoa"
+version = "1.0.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"
+
+[[package]]
+name = "js-sys"
+version = "0.3.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"
+dependencies = [
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "kuchiki"
+version = "0.8.1"
+dependencies = [
+ "cssparser",
+ "html5ever",
+ "indexmap 1.9.3",
+ "matches",
+ "selectors",
+ "thiserror",
+]
+
+[[package]]
+name = "leechy"
+version = "0.1.0"
+dependencies = [
+ "anyhow",
+ "kuchiki",
+ "rand",
+ "reqwest",
+ "serde",
+ "toml",
+]
+
+[[package]]
+name = "leechy"
+version = "0.1.0+py"
+dependencies = [
+ "leechy 0.1.0",
+ "pyo3",
+ "tokio",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.155"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"
+
+[[package]]
+name = "linux-raw-sys"
+version = "0.4.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"
+
+[[package]]
+name = "lock_api"
+version = "0.4.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"
+dependencies = [
+ "autocfg",
+ "scopeguard",
+]
+
+[[package]]
+name = "log"
+version = "0.4.22"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"
+
+[[package]]
+name = "mac"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
+
+[[package]]
+name = "markup5ever"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016"
+dependencies = [
+ "log",
+ "phf 0.10.1",
+ "phf_codegen",
+ "string_cache",
+ "string_cache_codegen",
+ "tendril",
+]
+
+[[package]]
+name = "matches"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"
+
+[[package]]
+name = "memchr"
+version = "2.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"
+
+[[package]]
+name = "memoffset"
+version = "0.9.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "mime"
+version = "0.3.17"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
+
+[[package]]
+name = "miniz_oxide"
+version = "0.7.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08"
+dependencies = [
+ "adler",
+]
+
+[[package]]
+name = "mio"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4"
+dependencies = [
+ "hermit-abi",
+ "libc",
+ "wasi",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "native-tls"
+version = "0.2.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466"
+dependencies = [
+ "libc",
+ "log",
+ "openssl",
+ "openssl-probe",
+ "openssl-sys",
+ "schannel",
+ "security-framework",
+ "security-framework-sys",
+ "tempfile",
+]
+
+[[package]]
+name = "new_debug_unreachable"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"
+
+[[package]]
+name = "object"
+version = "0.36.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"
+
+[[package]]
+name = "openssl"
+version = "0.10.66"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"
+dependencies = [
+ "bitflags 2.6.0",
+ "cfg-if",
+ "foreign-types",
+ "libc",
+ "once_cell",
+ "openssl-macros",
+ "openssl-sys",
+]
+
+[[package]]
+name = "openssl-macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.72",
+]
+
+[[package]]
+name = "openssl-probe"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
+
+[[package]]
+name = "openssl-sys"
+version = "0.9.103"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6"
+dependencies = [
+ "cc",
+ "libc",
+ "pkg-config",
+ "vcpkg",
+]
+
+[[package]]
+name = "parking_lot"
+version = "0.12.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"
+dependencies = [
+ "lock_api",
+ "parking_lot_core",
+]
+
+[[package]]
+name = "parking_lot_core"
+version = "0.9.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"
+dependencies = [
+ "cfg-if",
+ "libc",
+ "redox_syscall",
+ "smallvec",
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "percent-encoding"
+version = "2.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"
+
+[[package]]
+name = "phf"
+version = "0.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259"
+dependencies = [
+ "phf_shared 0.10.0",
+]
+
+[[package]]
+name = "phf"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"
+dependencies = [
+ "phf_macros",
+ "phf_shared 0.11.2",
+]
+
+[[package]]
+name = "phf_codegen"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd"
+dependencies = [
+ "phf_generator 0.10.0",
+ "phf_shared 0.10.0",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"
+dependencies = [
+ "phf_shared 0.10.0",
+ "rand",
+]
+
+[[package]]
+name = "phf_generator"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0"
+dependencies = [
+ "phf_shared 0.11.2",
+ "rand",
+]
+
+[[package]]
+name = "phf_macros"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b"
+dependencies = [
+ "phf_generator 0.11.2",
+ "phf_shared 0.11.2",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.72",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"
+dependencies = [
+ "siphasher",
+]
+
+[[package]]
+name = "phf_shared"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b"
+dependencies = [
+ "siphasher",
+]
+
+[[package]]
+name = "pin-project-lite"
+version = "0.2.14"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"
+
+[[package]]
+name = "pin-utils"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.30"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"
+
+[[package]]
+name = "portable-atomic"
+version = "1.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265"
+
+[[package]]
+name = "ppv-lite86"
+version = "0.2.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"
+dependencies = [
+ "zerocopy",
+]
+
+[[package]]
+name = "precomputed-hash"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.86"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"
+dependencies = [
+ "unicode-ident",
+]
+
+[[package]]
+name = "pyo3"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "831e8e819a138c36e212f3af3fd9eeffed6bf1510a805af35b0edee5ffa59433"
+dependencies = [
+ "cfg-if",
+ "indoc",
+ "libc",
+ "memoffset",
+ "once_cell",
+ "portable-atomic",
+ "pyo3-build-config",
+ "pyo3-ffi",
+ "pyo3-macros",
+ "unindent",
+]
+
+[[package]]
+name = "pyo3-build-config"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e8730e591b14492a8945cdff32f089250b05f5accecf74aeddf9e8272ce1fa8"
+dependencies = [
+ "once_cell",
+ "target-lexicon",
+]
+
+[[package]]
+name = "pyo3-ffi"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5e97e919d2df92eb88ca80a037969f44e5e70356559654962cbb3316d00300c6"
+dependencies = [
+ "libc",
+ "pyo3-build-config",
+]
+
+[[package]]
+name = "pyo3-macros"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb57983022ad41f9e683a599f2fd13c3664d7063a3ac5714cae4b7bee7d3f206"
+dependencies = [
+ "proc-macro2",
+ "pyo3-macros-backend",
+ "quote",
+ "syn 2.0.72",
+]
+
+[[package]]
+name = "pyo3-macros-backend"
+version = "0.22.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec480c0c51ddec81019531705acac51bcdbeae563557c982aa8263bb96880372"
+dependencies = [
+ "heck",
+ "proc-macro2",
+ "pyo3-build-config",
+ "quote",
+ "syn 2.0.72",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.36"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rand"
+version = "0.8.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
+dependencies = [
+ "libc",
+ "rand_chacha",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_chacha"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
+dependencies = [
+ "ppv-lite86",
+ "rand_core",
+]
+
+[[package]]
+name = "rand_core"
+version = "0.6.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
+dependencies = [
+ "getrandom",
+]
+
+[[package]]
+name = "redox_syscall"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4"
+dependencies = [
+ "bitflags 2.6.0",
+]
+
+[[package]]
+name = "reqwest"
+version = "0.11.27"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"
+dependencies = [
+ "base64",
+ "bytes",
+ "encoding_rs",
+ "futures-core",
+ "futures-util",
+ "h2",
+ "http",
+ "http-body",
+ "hyper",
+ "hyper-tls",
+ "ipnet",
+ "js-sys",
+ "log",
+ "mime",
+ "native-tls",
+ "once_cell",
+ "percent-encoding",
+ "pin-project-lite",
+ "rustls-pemfile",
+ "serde",
+ "serde_json",
+ "serde_urlencoded",
+ "sync_wrapper",
+ "system-configuration",
+ "tokio",
+ "tokio-native-tls",
+ "tokio-util",
+ "tower-service",
+ "url",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "wasm-streams",
+ "web-sys",
+ "winreg",
+]
+
+[[package]]
+name = "rustc-demangle"
+version = "0.1.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"
+
+[[package]]
+name = "rustix"
+version = "0.38.34"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"
+dependencies = [
+ "bitflags 2.6.0",
+ "errno",
+ "libc",
+ "linux-raw-sys",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "rustls-pemfile"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"
+dependencies = [
+ "base64",
+]
+
+[[package]]
+name = "ryu"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"
+
+[[package]]
+name = "schannel"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534"
+dependencies = [
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "scopeguard"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
+
+[[package]]
+name = "security-framework"
+version = "2.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"
+dependencies = [
+ "bitflags 2.6.0",
+ "core-foundation",
+ "core-foundation-sys",
+ "libc",
+ "security-framework-sys",
+]
+
+[[package]]
+name = "security-framework-sys"
+version = "2.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "selectors"
+version = "0.25.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06"
+dependencies = [
+ "bitflags 2.6.0",
+ "cssparser",
+ "derive_more",
+ "fxhash",
+ "log",
+ "new_debug_unreachable",
+ "phf 0.10.1",
+ "phf_codegen",
+ "precomputed-hash",
+ "servo_arc",
+ "smallvec",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.205"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.205"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.72",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.122"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"
+dependencies = [
+ "itoa",
+ "memchr",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "serde_spanned"
+version = "0.6.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "serde_urlencoded"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
+dependencies = [
+ "form_urlencoded",
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "servo_arc"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d036d71a959e00c77a63538b90a6c2390969f9772b096ea837205c6bd0491a44"
+dependencies = [
+ "stable_deref_trait",
+]
+
+[[package]]
+name = "signal-hook-registry"
+version = "1.4.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "siphasher"
+version = "0.3.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"
+
+[[package]]
+name = "slab"
+version = "0.4.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
+name = "smallvec"
+version = "1.13.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"
+
+[[package]]
+name = "socket2"
+version = "0.5.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c"
+dependencies = [
+ "libc",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "stable_deref_trait"
+version = "1.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
+
+[[package]]
+name = "string_cache"
+version = "0.8.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b"
+dependencies = [
+ "new_debug_unreachable",
+ "once_cell",
+ "parking_lot",
+ "phf_shared 0.10.0",
+ "precomputed-hash",
+ "serde",
+]
+
+[[package]]
+name = "string_cache_codegen"
+version = "0.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988"
+dependencies = [
+ "phf_generator 0.10.0",
+ "phf_shared 0.10.0",
+ "proc-macro2",
+ "quote",
+]
+
+[[package]]
+name = "syn"
+version = "1.0.109"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.72"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "sync_wrapper"
+version = "0.1.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"
+
+[[package]]
+name = "system-configuration"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"
+dependencies = [
+ "bitflags 1.3.2",
+ "core-foundation",
+ "system-configuration-sys",
+]
+
+[[package]]
+name = "system-configuration-sys"
+version = "0.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"
+dependencies = [
+ "core-foundation-sys",
+ "libc",
+]
+
+[[package]]
+name = "target-lexicon"
+version = "0.12.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"
+
+[[package]]
+name = "tempfile"
+version = "3.12.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"
+dependencies = [
+ "cfg-if",
+ "fastrand",
+ "once_cell",
+ "rustix",
+ "windows-sys 0.59.0",
+]
+
+[[package]]
+name = "tendril"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"
+dependencies = [
+ "futf",
+ "mac",
+ "utf-8",
+]
+
+[[package]]
+name = "thiserror"
+version = "1.0.63"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724"
+dependencies = [
+ "thiserror-impl",
+]
+
+[[package]]
+name = "thiserror-impl"
+version = "1.0.63"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.72",
+]
+
+[[package]]
+name = "tinyvec"
+version = "1.8.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938"
+dependencies = [
+ "tinyvec_macros",
+]
+
+[[package]]
+name = "tinyvec_macros"
+version = "0.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"
+
+[[package]]
+name = "tokio"
+version = "1.39.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1"
+dependencies = [
+ "backtrace",
+ "bytes",
+ "libc",
+ "mio",
+ "parking_lot",
+ "pin-project-lite",
+ "signal-hook-registry",
+ "socket2",
+ "tokio-macros",
+ "windows-sys 0.52.0",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.72",
+]
+
+[[package]]
+name = "tokio-native-tls"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"
+dependencies = [
+ "native-tls",
+ "tokio",
+]
+
+[[package]]
+name = "tokio-util"
+version = "0.7.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1"
+dependencies = [
+ "bytes",
+ "futures-core",
+ "futures-sink",
+ "pin-project-lite",
+ "tokio",
+]
+
+[[package]]
+name = "toml"
+version = "0.8.19"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"
+dependencies = [
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "toml_edit",
+]
+
+[[package]]
+name = "toml_datetime"
+version = "0.6.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.22.20"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"
+dependencies = [
+ "indexmap 2.3.0",
+ "serde",
+ "serde_spanned",
+ "toml_datetime",
+ "winnow",
+]
+
+[[package]]
+name = "tower-service"
+version = "0.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"
+
+[[package]]
+name = "tracing"
+version = "0.1.40"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"
+dependencies = [
+ "pin-project-lite",
+ "tracing-core",
+]
+
+[[package]]
+name = "tracing-core"
+version = "0.1.32"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"
+dependencies = [
+ "once_cell",
+]
+
+[[package]]
+name = "try-lock"
+version = "0.2.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
+
+[[package]]
+name = "unicode-bidi"
+version = "0.3.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"
+
+[[package]]
+name = "unicode-ident"
+version = "1.0.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
+
+[[package]]
+name = "unicode-normalization"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"
+dependencies = [
+ "tinyvec",
+]
+
+[[package]]
+name = "unindent"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"
+
+[[package]]
+name = "url"
+version = "2.5.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"
+dependencies = [
+ "form_urlencoded",
+ "idna",
+ "percent-encoding",
+]
+
+[[package]]
+name = "utf-8"
+version = "0.7.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"
+
+[[package]]
+name = "vcpkg"
+version = "0.2.15"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"
+
+[[package]]
+name = "want"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
+dependencies = [
+ "try-lock",
+]
+
+[[package]]
+name = "wasi"
+version = "0.11.0+wasi-snapshot-preview1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"
+
+[[package]]
+name = "wasm-bindgen"
+version = "0.2.92"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"
+dependencies = [
+ "cfg-if",
+ "wasm-bindgen-macro",
+]
+
+[[package]]
+name = "wasm-bindgen-backend"
+version = "0.2.92"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"
+dependencies = [
+ "bumpalo",
+ "log",
+ "once_cell",
+ "proc-macro2",
+ "quote",
+ "syn 2.0.72",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-futures"
+version = "0.4.42"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"
+dependencies = [
+ "cfg-if",
+ "js-sys",
+ "wasm-bindgen",
+ "web-sys",
+]
+
+[[package]]
+name = "wasm-bindgen-macro"
+version = "0.2.92"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"
+dependencies = [
+ "quote",
+ "wasm-bindgen-macro-support",
+]
+
+[[package]]
+name = "wasm-bindgen-macro-support"
+version = "0.2.92"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.72",
+ "wasm-bindgen-backend",
+ "wasm-bindgen-shared",
+]
+
+[[package]]
+name = "wasm-bindgen-shared"
+version = "0.2.92"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"
+
+[[package]]
+name = "wasm-streams"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129"
+dependencies = [
+ "futures-util",
+ "js-sys",
+ "wasm-bindgen",
+ "wasm-bindgen-futures",
+ "web-sys",
+]
+
+[[package]]
+name = "web-sys"
+version = "0.3.69"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"
+dependencies = [
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"
+dependencies = [
+ "windows-targets 0.48.5",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.52.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-sys"
+version = "0.59.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
+dependencies = [
+ "windows-targets 0.52.6",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"
+dependencies = [
+ "windows_aarch64_gnullvm 0.48.5",
+ "windows_aarch64_msvc 0.48.5",
+ "windows_i686_gnu 0.48.5",
+ "windows_i686_msvc 0.48.5",
+ "windows_x86_64_gnu 0.48.5",
+ "windows_x86_64_gnullvm 0.48.5",
+ "windows_x86_64_msvc 0.48.5",
+]
+
+[[package]]
+name = "windows-targets"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
+dependencies = [
+ "windows_aarch64_gnullvm 0.52.6",
+ "windows_aarch64_msvc 0.52.6",
+ "windows_i686_gnu 0.52.6",
+ "windows_i686_gnullvm",
+ "windows_i686_msvc 0.52.6",
+ "windows_x86_64_gnu 0.52.6",
+ "windows_x86_64_gnullvm 0.52.6",
+ "windows_x86_64_msvc 0.52.6",
+]
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"
+
+[[package]]
+name = "windows_aarch64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"
+
+[[package]]
+name = "windows_aarch64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"
+
+[[package]]
+name = "windows_i686_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
+
+[[package]]
+name = "windows_i686_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"
+
+[[package]]
+name = "windows_i686_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"
+
+[[package]]
+name = "windows_x86_64_gnu"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"
+
+[[package]]
+name = "windows_x86_64_gnullvm"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.48.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"
+
+[[package]]
+name = "windows_x86_64_msvc"
+version = "0.52.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+
+[[package]]
+name = "winnow"
+version = "0.6.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "winreg"
+version = "0.50.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"
+dependencies = [
+ "cfg-if",
+ "windows-sys 0.48.0",
+]
+
+[[package]]
+name = "zerocopy"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"
+dependencies = [
+ "byteorder",
+ "zerocopy-derive",
+]
+
+[[package]]
+name = "zerocopy-derive"
+version = "0.7.35"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.72",
+]
diff --git a/crates/leechy-py/Cargo.toml b/crates/leechy-py/Cargo.toml
new file mode 100644
index 00000000..f00af3d5
--- /dev/null
+++ b/crates/leechy-py/Cargo.toml
@@ -0,0 +1,17 @@
+[package]
+edition = "2021"
+license = "AGPL-3.0"
+name = "leechy"
+version = "0.1.0+py"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+crate-type = ["cdylib"]
+name = "leechy"
+
+[dependencies]
+leechy = {path = "../leechy"}
+pyo3 = {version = "0.22.0", features = ["extension-module", "experimental-async"]}
+tokio = {version = "1.23.1", features = ["full"]}
+
+[workspace]
diff --git a/crates/leechy-py/pyproject.toml b/crates/leechy-py/pyproject.toml
new file mode 100644
index 00000000..14fde22c
--- /dev/null
+++ b/crates/leechy-py/pyproject.toml
@@ -0,0 +1,15 @@
+[build-system]
+build-backend = "maturin"
+requires = ["maturin>=1.7,<2.0"]
+
+[project]
+classifiers = [
+ "Programming Language :: Rust",
+ "Programming Language :: Python :: Implementation :: CPython",
+ "Programming Language :: Python :: Implementation :: PyPy",
+]
+dynamic = ["version"]
+name = "leechy"
+requires-python = ">=3.8"
+[tool.maturin]
+features = ["pyo3/extension-module"]
diff --git a/crates/leechy-py/src/lib.rs b/crates/leechy-py/src/lib.rs
new file mode 100644
index 00000000..1cf8b66a
--- /dev/null
+++ b/crates/leechy-py/src/lib.rs
@@ -0,0 +1,61 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use pyo3::prelude::*;
+
+static TOKIO_RUNTIME: std::sync::LazyLock<tokio::runtime::Runtime> =
+ std::sync::LazyLock::new(|| {
+ tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .build()
+ .unwrap()
+ });
+
+#[pymodule]
+mod leechy {
+ use super::*;
+ use ::leechy as lchy;
+
+ #[pyclass]
+ struct Engine {
+ inner: lchy::Engine,
+ }
+
+ #[pymethods]
+ impl Engine {
+ #[new]
+ #[pyo3(signature = (name = "startpage"))]
+ fn new(name: &str) -> PyResult<Self> {
+ match lchy::Engine::by_name(name) {
+ Some(engine) => Ok(Self { inner: engine }),
+ None => Err(pyo3::exceptions::PyValueError::new_err(format!(
+ "Unknown engine: {}",
+ name
+ ))),
+ }
+ }
+
+ fn search(&self, query: String) -> PyResult<Vec<String>> {
+ TOKIO_RUNTIME.block_on(async {
+ self.inner
+ .search(query.as_str())
+ .await
+ .map_err(|e| pyo3::exceptions::PyException::new_err(format!("{}", e)))
+ .map(|res| res.into_iter().map(|r| r.to_string()).collect())
+ })
+ }
+ }
+}
diff --git a/crates/leechy/Cargo.toml b/crates/leechy/Cargo.toml
new file mode 100644
index 00000000..8662e20e
--- /dev/null
+++ b/crates/leechy/Cargo.toml
@@ -0,0 +1,15 @@
+[package]
+edition = "2021"
+license = "AGPL-3.0"
+name = "leechy"
+version = "0.1.0"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow.workspace = true
+kuchiki = {path = "../kuchiki"}
+rand.workspace = true
+reqwest.workspace = true
+serde.workspace = true
+toml.workspace = true
diff --git a/crates/leechy/src/conf.toml b/crates/leechy/src/conf.toml
new file mode 100644
index 00000000..1c492b01
--- /dev/null
+++ b/crates/leechy/src/conf.toml
@@ -0,0 +1,4 @@
+[[engines]]
+name = "startpage"
+search = "https://www.startpage.com/search?q={query}&qsr=en_US"
+xpath = "./div[contains(@class, 'result')]/a"
diff --git a/crates/leechy/src/engine.rs b/crates/leechy/src/engine.rs
new file mode 100644
index 00000000..7b4b939a
--- /dev/null
+++ b/crates/leechy/src/engine.rs
@@ -0,0 +1,91 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use kuchiki::traits::*;
+use std::sync::LazyLock;
+
+static PARSED_CONFIG: LazyLock<Config> =
+ LazyLock::new(|| toml::from_str(include_str!("conf.toml")).unwrap());
+
+#[derive(serde::Deserialize, Clone)]
+pub struct Config {
+ engines: Vec<EngineConf>,
+}
+
+impl Config {
+ pub fn new() -> Self {
+ PARSED_CONFIG.clone()
+ }
+}
+
+#[derive(serde::Deserialize, Clone)]
+pub struct EngineConf {
+ name: String,
+ search: String,
+ xpath: String,
+}
+
+pub struct Engine {
+ conf: EngineConf,
+}
+
+impl Engine {
+ pub fn new(conf: EngineConf) -> Self {
+ Engine { conf }
+ }
+
+ pub fn by_name(name: &str) -> Option<Self> {
+ let conf = Config::new().engines.into_iter().find(|e| e.name == name)?;
+ Some(Engine::new(conf))
+ }
+
+ pub async fn search(&self, query: &str) -> Result<Vec<Url>, reqwest::Error> {
+ let encoded_query = query.replace(' ', "+");
+ let url = self.conf.search.replace("{query}", &encoded_query);
+ let useragent = crate::useragent::UserAgent::random_weighted();
+
+ let client = reqwest::Client::builder()
+ .user_agent(useragent.as_str())
+ .build()?;
+
+ let body = client.get(&url).send().await?.text().await?;
+
+ Ok(kuchiki::parse_html()
+ .one(body)
+ .select_xpath(&self.conf.xpath)
+ .map(|url| {
+ let attr = url.attributes.borrow();
+ let url = attr.get("href").unwrap();
+ Url(url.to_string())
+ })
+ .collect())
+ }
+}
+
+#[derive(Debug)]
+pub struct Url(String);
+
+impl std::fmt::Display for Url {
+ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+ write!(f, "{}", self.0)
+ }
+}
+
+impl AsRef<str> for Url {
+ fn as_ref(&self) -> &str {
+ &self.0
+ }
+}
diff --git a/crates/leechy/src/lib.rs b/crates/leechy/src/lib.rs
new file mode 100644
index 00000000..0ab1eec4
--- /dev/null
+++ b/crates/leechy/src/lib.rs
@@ -0,0 +1,20 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+mod engine;
+mod useragent;
+
+pub use engine::Engine;
diff --git a/crates/leechy/src/useragent.rs b/crates/leechy/src/useragent.rs
new file mode 100644
index 00000000..c43a9aa1
--- /dev/null
+++ b/crates/leechy/src/useragent.rs
@@ -0,0 +1,58 @@
+// Stract is an open source web search engine.
+// Copyright (C) 2024 Stract ApS
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <https://www.gnu.org/licenses/>
+
+use rand::prelude::*;
+use std::sync::LazyLock;
+
+static USER_AGENTS: LazyLock<Vec<(usize, String)>> = LazyLock::new(|| {
+ include_str!("useragents.txt")
+ .lines()
+ .map(|line| line.trim().to_string())
+ .enumerate()
+ .collect()
+});
+
+pub struct UserAgent(String);
+
+impl UserAgent {
+ pub fn random_weighted() -> Self {
+ let mut rng = thread_rng();
+
+ UserAgent(
+ USER_AGENTS
+ .choose_weighted(&mut rng, |(rank, _)| 1 / (rank + 1))
+ .unwrap()
+ .1
+ .clone(),
+ )
+ }
+
+ pub fn as_str(&self) -> &str {
+ &self.0
+ }
+}
+
+impl std::fmt::Display for UserAgent {
+ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+ write!(f, "{}", self.0)
+ }
+}
+
+impl AsRef<str> for UserAgent {
+ fn as_ref(&self) -> &str {
+ &self.0
+ }
+}
diff --git a/crates/leechy/src/useragents.txt b/crates/leechy/src/useragents.txt
new file mode 100644
index 00000000..104e96de
--- /dev/null
+++ b/crates/leechy/src/useragents.txt
@@ -0,0 +1,14 @@
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.1
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.3
+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.3
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.3
+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.3
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.3
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Unique/100.7.6266.6
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 OPR/112.0.0.
+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/117.
+Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.3
+Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.192 Safari/537.3
+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.3
\ No newline at end of file
diff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.py
index 718e42ea..2907cf27 100644
--- a/ltr/auto_annotate.py
+++ b/ltr/auto_annotate.py
@@ -7,41 +7,18 @@ import json
import time
import requests
import random
-
-# API = "https://stract.com/beta/api/search"
-API = "http://localhost:3000/beta/api/search"
-NUM_RESULTS_PER_QUERY = 20
+from db import Db
+import stract
ELO_K = 32
ELO_SCALE = 400
ELO_ROUNDS_MULT = 5
NUM_LABELS = 4
-# PROMPT = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>
-#
-# You are a helpful, smart, kind, and efficient AI assistant. You always fulfill the user's requests to the best of your ability.<|eot_id|><|start_header_id|>user<|end_header_id|>
-#
-# Think about this step-by-step. You are a search engine evaluator and your task is to evaluate search results based on how well the result matches the query
-# You will be shown two results for each query and most choose which result is best for the users query. A good result most answer the users query and come from an authoritative source.
-# To choose the best result, write "Best: RESULT_A" or "Best: RESULT_B". Before choosing the best result, you should first evaluate the relevance of each result to the query.
-#
-# Query: "{}"
-#
-# RESULT_A:
-# Url: "{}"
-# Title: "{}"
-# Snippet: "{}"
-#
-# RESULT_B:
-# Url: "{}"
-# Title: "{}"
-# Snippet: "{}"
-#
-# Evaluation:<|eot_id|><|start_header_id|>assistant<|end_header_id|>
-#
-# """
-
-PROMPT = """<start_of_turn>user
+PROMPT = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>
+
+You are a helpful, smart, kind, and efficient AI assistant. You always fulfill the user's requests to the best of your ability.<|eot_id|><|start_header_id|>user<|end_header_id|>
+
Think about this step-by-step. You are a search engine evaluator and your task is to evaluate search results based on how well the result matches the query
You will be shown two results for each query and most choose which result is best for the users query. A good result most answer the users query and come from an authoritative source.
To choose the best result, write "Best: RESULT_A" or "Best: RESULT_B". Before choosing the best result, you should first evaluate the relevance of each result to the query.
@@ -56,14 +33,16 @@ Snippet: "{}"
RESULT_B:
Url: "{}"
Title: "{}"
-Snippet: "{}"<end_of_turn>
-<start_of_turn>model
+Snippet: "{}"
+
+Evaluation:<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+
"""
+
llm = Llama(
n_gpu_layers=-1,
n_ctx=8000,
- # model_path="data/Meta-Llama-3.1-8B-Instruct-Q8_0.gguf",
- model_path="data/gemma-2-2b-it-abliterated-Q6_K_L.gguf",
+ model_path="data/Meta-Llama-3.1-8B-Instruct-Q8_0.gguf",
repeat_penalty=False,
no_penalize_nl=True,
verbose=False,
@@ -77,39 +56,8 @@ with open("data/queries_us.csv") as f:
np.random.shuffle(all_queries)
-def setup_db():
- db = sqlite3.connect("data/auto-ranking-annotation.sqlite")
-
- cur = db.cursor()
-
- cur.execute(
- """
- CREATE TABLE IF NOT EXISTS queries (
- qid UUID PRIMARY KEY DEFAULT (HEX(RANDOMBLOB(16))),
- query TEXT NOT NULL UNIQUE
- );"""
- )
-
- cur.execute(
- """
- CREATE TABLE IF NOT EXISTS search_results (
- qid UUID NOT NULL,
- url TEXT NOT NULL,
- orig_rank INTEGER NOT NULL,
- webpage_json TEXT NOT NULL,
- annotation INTEGER DEFAULT NULL,
- PRIMARY KEY (qid, url)
- );"""
- )
+db = Db("data/auto-ranking-annotation.sqlite")
- db.commit()
-
- return db
-
-
-db = setup_db()
-
-cur = db.cursor()
for query in all_queries:
if len(query) < 3:
continue
@@ -125,82 +73,15 @@ for query in all_queries:
if len(query) > 100:
continue
- cur.execute(
- """
- INSERT OR IGNORE INTO queries (query)
- VALUES (?)
- """,
- (query,),
- )
+ db.add_query(query)
-db.commit()
-unannotated_queries = cur.execute(
- """
- SELECT qid, query
- FROM queries
- WHERE NOT EXISTS (
- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL
- )
- ORDER BY qid
- """,
-).fetchall()
-
-unannotated_queries = {qid: query for qid, query in unannotated_queries}
-
-
-def simplify_snippet(snippet):
- if "text" not in snippet:
- return ""
-
- return "".join([f["text"] for f in snippet["text"]["fragments"]])
-
-
-def get_search_results(query):
- payload = {
- "query": query,
- "numResults": NUM_RESULTS_PER_QUERY,
- "returnRankingSignals": True,
- 'signalCoefficients': {
- 'host_centrality_rank': 0.022,
- 'page_centrality_rank': 0.022,
- }
- }
-
- return [
- {
- "url": w["url"],
- "title": w["title"],
- "rankingSignals": {s: v["value"] for (s, v) in w["rankingSignals"].items()},
- "snippet": simplify_snippet(w["snippet"]),
- }
- for w in requests.post(API, json=payload).json()["webpages"][
- :NUM_RESULTS_PER_QUERY
- ]
- ]
-
-
-def add_results(query):
- has_results = (
- cur.execute("SELECT 1 FROM search_results WHERE qid = ?", (qid,)).fetchone()
- is not None
- )
- if has_results:
- return
+unannotated_queries = db.get_unannotated_queries()
- results = get_search_results(query)
+def add_results(qid, query):
+ results = stract.search(query)
time.sleep(1)
-
- for i, result in enumerate(results):
- cur.execute(
- """
- INSERT OR IGNORE INTO search_results (qid, url, orig_rank, webpage_json)
- VALUES (?, ?, ?, ?)
- """,
- (qid, result["url"], i, json.dumps(result)),
- )
-
- db.commit()
+ db.insert_results(qid, results)
def get_prompt(query, url_a, title_a, snippet_a, url_b, title_b, snippet_b):
@@ -233,16 +114,8 @@ def elo_update(winner, loser, elo):
for qid, query in tqdm(unannotated_queries.items()):
- add_results(query)
- unnanotated_results = cur.execute(
- """
- SELECT url, orig_rank, webpage_json
- FROM search_results
- WHERE qid = ? AND annotation IS NULL
- ORDER BY orig_rank
- """,
- (qid,),
- ).fetchall()
+ add_results(qid, query)
+ unnanotated_results = db.get_unannotated_results(qid)
elo = {url: ELO_SCALE // 2 for url, _, _ in unnanotated_results}
@@ -274,7 +147,6 @@ for qid, query in tqdm(unannotated_queries.items()):
elo = [{"url": url} for url, _ in elo]
for i in range(len(elo)):
- # elo[i]['label'] = (NUM_LABELS - i // (len(elo) // NUM_LABELS))
elo[i]['label'] = NUM_LABELS - int(np.log2(i + 1))
print(query)
@@ -282,12 +154,4 @@ for qid, query in tqdm(unannotated_queries.items()):
for website in elo:
url = website['url']
relevancy = website['label']
- cur.execute(
- """
- UPDATE search_results
- SET annotation = ?
- WHERE qid = ? AND url = ?
- """,
- (relevancy, qid, url),
- )
- db.commit()
+ db.annotate(qid, url, relevancy)
diff --git a/ltr/db.py b/ltr/db.py
new file mode 100644
index 00000000..0fad1688
--- /dev/null
+++ b/ltr/db.py
@@ -0,0 +1,98 @@
+import sqlite3
+import json
+
+class Db:
+ def __init__(self, db_path):
+ self.db_path = db_path
+ self.conn = sqlite3.connect(db_path)
+ self.cursor = self.conn.cursor()
+ self.setup_db()
+
+ def __del__(self):
+ self.cursor.close()
+ self.conn.close()
+
+ def setup_db(self):
+ self.cursor.execute(
+ """
+ CREATE TABLE IF NOT EXISTS queries (
+ qid UUID PRIMARY KEY DEFAULT (HEX(RANDOMBLOB(16))),
+ query TEXT NOT NULL UNIQUE
+ );"""
+ )
+
+ self.cursor.execute(
+ """
+ CREATE TABLE IF NOT EXISTS search_results (
+ qid UUID NOT NULL,
+ url TEXT NOT NULL,
+ orig_rank INTEGER NOT NULL,
+ webpage_json TEXT NOT NULL,
+ annotation INTEGER DEFAULT NULL,
+ PRIMARY KEY (qid, url)
+ );"""
+ )
+
+ self.conn.commit()
+
+ def add_query(self, query: str):
+ self.cursor.execute(
+ """
+ INSERT OR IGNORE INTO queries (query)
+ VALUES (?)
+ """,
+ (query,),
+ )
+ self.conn.commit()
+
+ def get_unannotated_queries(self):
+ unannotated_queries = self.cursor.execute(
+ """
+ SELECT qid, query
+ FROM queries
+ WHERE NOT EXISTS (
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL
+ )
+ ORDER BY qid
+ """,
+ ).fetchall()
+
+ return {qid: query for qid, query in unannotated_queries}
+
+ def insert_result(self, qid, rank, result):
+ self.cursor.execute(
+ """
+ INSERT OR IGNORE INTO search_results (qid, url, orig_rank, webpage_json)
+ VALUES (?, ?, ?, ?)
+ """,
+ (qid, result["url"], rank, json.dumps(result)),
+ )
+
+ self.conn.commit()
+
+ def insert_results(self, qid, results):
+ for i, result in enumerate(results):
+ self.insert_result(qid, i, result)
+
+
+ def get_unannotated_results(self, qid):
+ return self.cursor.execute(
+ """
+ SELECT url, orig_rank, webpage_json
+ FROM search_results
+ WHERE qid = ? AND annotation IS NULL
+ ORDER BY orig_rank
+ """,
+ (qid,),
+ ).fetchall()
+
+ def annotate(self, qid, url, annotation):
+ self.cursor.execute(
+ """
+ UPDATE search_results
+ SET annotation = ?
+ WHERE qid = ? AND url = ?
+ """,
+ (annotation, qid, url),
+ )
+ self.conn.commit()
diff --git a/ltr/lambdamart.py b/ltr/lambdamart.py
index ed3026eb..26a27a30 100644
--- a/ltr/lambdamart.py
+++ b/ltr/lambdamart.py
@@ -29,7 +29,7 @@ res = cur.execute(
SELECT qid, query
FROM queries
WHERE EXISTS (
- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation > 0
)
"""
)
diff --git a/ltr/leechy_annotate.py b/ltr/leechy_annotate.py
new file mode 100644
index 00000000..f1443efd
--- /dev/null
+++ b/ltr/leechy_annotate.py
@@ -0,0 +1,71 @@
+import leechy
+from db import Db
+import stract
+import numpy as np
+from tqdm import tqdm
+import time
+
+NUM_LABELS = 4
+
+with open("data/queries_us.csv") as f:
+ all_queries = [line.strip() for line in f.readlines()]
+
+# shuffle queries
+np.random.shuffle(all_queries)
+
+db = Db("data/auto-ranking-annotation.sqlite")
+
+for query in all_queries:
+ if len(query) < 3:
+ continue
+
+ # check if query has large percentage of non-alphanumeric characters
+ if sum([c.isalnum() for c in query]) / len(query) < 0.5:
+ continue
+
+ if len(query) > 100:
+ continue
+
+ db.add_query(query)
+
+
+unannotated_queries = db.get_unannotated_queries()
+
+eng = leechy.Engine()
+for qid, query in tqdm(unannotated_queries.items()):
+ tqdm.write(query)
+ leechy_results = eng.search(query)
+
+ for i, result in enumerate(leechy_results):
+ label = NUM_LABELS - int(np.log2(i + 1))
+
+ try:
+ data = next(
+ res
+ for res in stract.search(f"{query} site:{result}")
+ if res["url"] == result
+ )
+ except StopIteration:
+ continue
+
+ if not data:
+ continue
+
+ tqdm.write(f"{result}: {label}")
+ db.insert_result(qid, i, data)
+ db.annotate(qid, result, label)
+
+ for page in range(2, 2 + 1):
+ bad_results = stract.search(query, page)
+
+ for i, result in enumerate(bad_results):
+ if result["url"] in leechy_results:
+ continue
+
+ label = 0
+ tqdm.write(f'{result["url"]}: {label}')
+ db.insert_result(qid, (i + 1) * page, result)
+ db.annotate(qid, result["url"], label)
+
+ time.sleep(np.random.normal(60, 20))
+ tqdm.write("")
diff --git a/ltr/linear_model.py b/ltr/linear_model.py
index ec2e8346..4a701db9 100644
--- a/ltr/linear_model.py
+++ b/ltr/linear_model.py
@@ -20,7 +20,7 @@ res = cur.execute(
SELECT qid, query
FROM queries
WHERE EXISTS (
- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL
+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation > 0
)
"""
)
@@ -108,14 +108,18 @@ y_test = jnp.array(y_test)
# jax model
w = jnp.zeros(X_train.shape[1])
+
def model(w, X):
return jnp.dot(X, w)
+
def loss(w, batch):
features, labels, mask = batch
- scores = model(w, features)
+ # scores = model(w, features)
+ # return rax.approx_t12n(rax.mrr_metric)(scores, labels)
+
+ return ((w * features - jnp.ones_like(w)) ** 2).sum()
- return rax.approx_t12n(rax.mrr_metric)(scores, labels)
grad_fn = jax.jit(jax.grad(loss))
@@ -123,12 +127,15 @@ grad_fn = jax.jit(jax.grad(loss))
for i in range(128):
batch_size = 20
for j in range(0, len(X_train), batch_size):
- batch = (X_train[j : j + batch_size], y_train[j : j + batch_size], jnp.ones(batch_size))
+ batch = (
+ X_train[j : j + batch_size],
+ y_train[j : j + batch_size],
+ jnp.ones(batch_size),
+ )
w = w - grad_fn(w, batch) * 3e-4
w = w.clip(0, 10)
-
# model = LinearRegression(fit_intercept=False, positive=True)
# model = SVR(kernel="linear")
# model = DecisionTreeRegressor()
@@ -136,7 +143,9 @@ for i in range(128):
print("TRAIN")
for k in [1, 2, 5, 10]:
- print(f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_train], [model(w, X_train)], k=k))
+ print(
+ f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_train], [model(w, X_train)], k=k)
+ )
print()
print("TEST")
@@ -144,7 +153,6 @@ for k in [1, 2, 5, 10]:
print(f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_test], [model(w, X_test)], k=k))
-
weights = {id2feature[i]: float(v) for (i, v) in enumerate(w)}
print()
print("Weights:")
diff --git a/ltr/stract.py b/ltr/stract.py
new file mode 100644
index 00000000..25026f27
--- /dev/null
+++ b/ltr/stract.py
@@ -0,0 +1,32 @@
+import requests
+
+# API = "https://stract.com/beta/api/search"
+API = "http://localhost:3000/beta/api/search"
+NUM_RESULTS_PER_QUERY = 20
+
+def simplify_snippet(snippet):
+ if "text" not in snippet:
+ return ""
+
+ return "".join([f["text"] for f in snippet["text"]["fragments"]])
+
+
+def search(query, page=0):
+ payload = {
+ "query": query,
+ "numResults": NUM_RESULTS_PER_QUERY,
+ 'page': page,
+ "returnRankingSignals": True,
+ }
+
+ return [
+ {
+ "url": w["url"],
+ "title": w["title"],
+ "rankingSignals": {s: v["value"] for (s, v) in w["rankingSignals"].items()},
+ "snippet": simplify_snippet(w["snippet"]),
+ }
+ for w in requests.post(API, json=payload).json()["webpages"][
+ :NUM_RESULTS_PER_QUERY
+ ]
+ ]
\ No newline at end of file