web search done right
leechy ranking annotation experiment
Mikkel Denkerf3315c4b422b
diff --git a/Cargo.lock b/Cargo.lockindex 4aa7859e..60b6d1de 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -103,9 +103,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"[[package]]name = "anstream"-version = "0.6.14"+version = "0.6.15"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b"+checksum = "64e15c1ab1f89faffbf04a634d5e1962e9074f2741eef6d97f3c4e322426d526"dependencies = ["anstyle","anstyle-parse",@@ -118,33 +118,33 @@ dependencies = [[[package]]name = "anstyle"-version = "1.0.7"+version = "1.0.8"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b"+checksum = "1bec1de6f59aedf83baf9ff929c98f2ad654b97c9510f4e70cf6f661d49fd5b1"[[package]]name = "anstyle-parse"-version = "0.2.4"+version = "0.2.5"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4"+checksum = "eb47de1e80c2b463c735db5b217a0ddc39d612e7ac9e2e96a5aed1f57616c1cb"dependencies = ["utf8parse",][[package]]name = "anstyle-query"-version = "1.1.0"+version = "1.1.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391"+checksum = "6d36fc52c7f6c869915e99412912f22093507da8d9e942ceaf66fe4b7c14422a"dependencies = ["windows-sys 0.52.0",][[package]]name = "anstyle-wincon"-version = "3.0.3"+version = "3.0.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19"+checksum = "5bf74e1b6e971609db8ca7a9ce79fd5768ab6ae46441c572e46cf596f59e57f8"dependencies = ["anstyle","windows-sys 0.52.0",@@ -598,9 +598,9 @@ checksum = "5ce89b21cab1437276d2650d57e971f9d548a2d9037cc231abdc0562b97498ce"[[package]]name = "bytemuck"-version = "1.16.1"+version = "1.16.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b236fc92302c97ed75b38da1f4917b5cdda4984745740f153a5d3059e48d725e"+checksum = "102087e286b4677862ea56cf8fc58bb2cdfa8725c40ffb80fe3a008eb7f2fc83"dependencies = ["bytemuck_derive",]@@ -636,9 +636,9 @@ checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495"[[package]]name = "bytes"-version = "1.6.1"+version = "1.7.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "a12916984aab3fa6e39d655a33e09c0071eb36d6ab3aea5c2d78551f1df6d952"+checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50"[[package]]name = "bzip2"@@ -746,9 +746,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5"[[package]]name = "cc"-version = "1.1.6"+version = "1.1.8"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2aba8f4e9906c7ce3c73463f62a7f0c65183ada1a2d47e397cc8810827f9694f"+checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549"dependencies = ["jobserver","libc",@@ -857,9 +857,9 @@ dependencies = [[[package]]name = "clap"-version = "4.5.10"+version = "4.5.14"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8f6b81fb3c84f5563d509c59b5a48d935f689e993afa90fe39047f05adef9142"+checksum = "c937d4061031a6d0c8da4b9a4f98a172fc2976dfb1c19213a9cf7d0d3c837e36"dependencies = ["clap_builder","clap_derive",@@ -867,9 +867,9 @@ dependencies = [[[package]]name = "clap_builder"-version = "4.5.10"+version = "4.5.14"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5ca6706fd5224857d9ac5eb9355f6683563cc0541c7cd9d014043b57cbec78ac"+checksum = "85379ba512b21a328adf887e85f7742d12e96eb31f3ef077df4ffc26b506ffed"dependencies = ["anstream","anstyle",@@ -879,9 +879,9 @@ dependencies = [[[package]]name = "clap_derive"-version = "4.5.8"+version = "4.5.13"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2bac35c6dafb060fd4d275d9a4ffae97917c13a6327903a8be2153cd964f7085"+checksum = "501d359d5f3dcaf6ecdeee48833ae73ec6e42723a1e52419c79abf9507eec0a0"dependencies = ["heck 0.5.0","proc-macro2",@@ -891,9 +891,9 @@ dependencies = [[[package]]name = "clap_lex"-version = "0.7.1"+version = "0.7.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70"+checksum = "1462739cb27611015575c0c11df5df7601141071f07518d56fcc1be504cbec97"[[package]]name = "client-wasm"@@ -923,9 +923,9 @@ checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b"[[package]]name = "colorchoice"-version = "1.0.1"+version = "1.0.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422"+checksum = "d3fd119d74b830634cea2a0f58bbd0d54540518a14397557951e79340abc28c0"[[package]]name = "console"@@ -1609,14 +1609,14 @@ dependencies = [[[package]]name = "filetime"-version = "0.2.23"+version = "0.2.24"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "1ee447700ac8aa0b2f2bd7bc4462ad686ba06baa6727ac149a2d6277f0d240fd"+checksum = "bf401df4a4e3872c4fe8151134cf483738e74b67fc934d6532c882b3d24a4550"dependencies = ["cfg-if","libc",- "redox_syscall 0.4.1",- "windows-sys 0.52.0",+ "libredox",+ "windows-sys 0.59.0",][[package]]@@ -1627,9 +1627,9 @@ checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80"[[package]]name = "flate2"-version = "1.0.30"+version = "1.0.31"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5f54427cfd1c7829e2a139fcefea601bf088ebca651d2bf53ebc600eac295dae"+checksum = "7f211bbe8e69bbd0cfdea405084f128ae8b4aaa6b0b522fc8f2b009084797920"dependencies = ["crc32fast","miniz_oxide",@@ -1990,7 +1990,7 @@ dependencies = ["futures-sink","futures-util","http 0.2.12",- "indexmap 2.2.6",+ "indexmap 2.3.0","slab","tokio","tokio-util",@@ -2279,9 +2279,9 @@ dependencies = [[[package]]name = "hyper-util"-version = "0.1.6"+version = "0.1.7"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3ab92f4f49ee4fb4f997c784b7a2e0fa70050211e0b6a287f898c3c9785ca956"+checksum = "cde7055719c54e36e95e8719f95883f22072a48ede39db7fc17a4e1d5281e9b9"dependencies = ["bytes","futures-channel",@@ -2397,9 +2397,9 @@ dependencies = [[[package]]name = "indexmap"-version = "2.2.6"+version = "2.3.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26"+checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0"dependencies = ["equivalent","hashbrown 0.14.5",@@ -2507,9 +2507,9 @@ dependencies = [[[package]]name = "is_terminal_polyfill"-version = "1.70.0"+version = "1.70.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800"+checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"[[package]]name = "itertools"@@ -2651,6 +2651,18 @@ version = "0.5.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "03087c2bad5e1034e8cace5926dec053fb3790248370865f5117a7d0213354c8"+[[package]]+name = "leechy"+version = "0.1.0"+dependencies = [+ "anyhow",+ "kuchiki",+ "rand",+ "reqwest 0.11.27",+ "serde",+ "toml",+]+[[package]]name = "lending-iter"version = "0.1.0"@@ -2692,6 +2704,7 @@ checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d"dependencies = ["bitflags 2.6.0","libc",+ "redox_syscall",][[package]]@@ -2765,9 +2778,9 @@ dependencies = [[[package]]name = "lru"-version = "0.12.3"+version = "0.12.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d3262e75e648fce39813cb56ac41f3c3e3f65217ebf3844d818d1f9398cfb0dc"+checksum = "37ee39891760e7d94734f6f63fedc29a2e4a152f836120753a72503f09fcf904"dependencies = ["hashbrown 0.14.5",]@@ -2871,7 +2884,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519"dependencies = ["cfg-if",- "rayon",][[package]]@@ -3157,11 +3169,11 @@ dependencies = [[[package]]name = "num_enum"-version = "0.7.2"+version = "0.7.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "02339744ee7253741199f897151b38e72257d13802d4ee837285cc2990a90845"+checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179"dependencies = [- "num_enum_derive 0.7.2",+ "num_enum_derive 0.7.3",][[package]]@@ -3178,9 +3190,9 @@ dependencies = [[[package]]name = "num_enum_derive"-version = "0.7.2"+version = "0.7.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "681030a937600a36906c185595136d26abfebb4aa9c65701cefcaf8578bb982b"+checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56"dependencies = ["proc-macro-crate 3.1.0","proc-macro2",@@ -3202,9 +3214,9 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"[[package]]name = "object"-version = "0.36.2"+version = "0.36.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "3f203fa8daa7bb185f760ae12bd8e097f63d17041dcdcaf675ac54cdf863170e"+checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9"dependencies = ["memchr",]@@ -3403,7 +3415,7 @@ checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"dependencies = ["cfg-if","libc",- "redox_syscall 0.5.3",+ "redox_syscall","smallvec","windows-targets 0.52.6",]@@ -3475,7 +3487,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db"dependencies = ["fixedbitset",- "indexmap 2.2.6",+ "indexmap 2.3.0",][[package]]@@ -3682,9 +3694,12 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391"[[package]]name = "ppv-lite86"-version = "0.2.17"+version = "0.2.20"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"+checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"+dependencies = [+ "zerocopy",+][[package]]name = "precomputed-hash"@@ -3865,16 +3880,17 @@ dependencies = [[[package]]name = "quinn"-version = "0.11.2"+version = "0.11.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e4ceeeeabace7857413798eb1ffa1e9c905a9946a57d81fb69b4b71c4d8eb3ad"+checksum = "b22d8e7369034b9a7132bc2008cac12f2013c8132b45e0554e6e20e2617f2156"dependencies = ["bytes","pin-project-lite","quinn-proto","quinn-udp",- "rustc-hash",+ "rustc-hash 2.0.0","rustls",+ "socket2","thiserror","tokio","tracing",@@ -3882,14 +3898,14 @@ dependencies = [[[package]]name = "quinn-proto"-version = "0.11.3"+version = "0.11.6"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ddf517c03a109db8100448a4be38d498df8a210a99fe0e1b9eaf39e78c640efe"+checksum = "ba92fb39ec7ad06ca2582c0ca834dfeadcaf06ddfc8e635c80aa7e1c05315fdd"dependencies = ["bytes","rand","ring",- "rustc-hash",+ "rustc-hash 2.0.0","rustls","slab","thiserror",@@ -3906,6 +3922,7 @@ dependencies = ["libc","once_cell","socket2",+ "tracing","windows-sys 0.52.0",]@@ -4019,16 +4036,15 @@ dependencies = [[[package]]name = "ravif"-version = "0.11.8"+version = "0.11.10"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "c6ba61c28ba24c0cf8406e025cb29a742637e3f70776e61c27a8a8b72a042d12"+checksum = "a8f0bfd976333248de2078d350bfdf182ff96e168a24d23d2436cef320dd4bdd"dependencies = ["avif-serialize","imgref","loop9","quick-error 2.0.1","rav1e",- "rayon","rgb",]@@ -4087,15 +4103,6 @@ dependencies = ["libc",]-[[package]]-name = "redox_syscall"-version = "0.4.1"-source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4722d768eff46b75989dd134e5c353f0d6296e5aaa3132e776cbdb56be7731aa"-dependencies = [- "bitflags 1.3.2",-]-[[package]]name = "redox_syscall"version = "0.5.3"@@ -4118,9 +4125,9 @@ dependencies = [[[package]]name = "regex"-version = "1.10.5"+version = "1.10.6"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f"+checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619"dependencies = ["aho-corasick 1.1.3","memchr",@@ -4234,7 +4241,7 @@ dependencies = ["pin-project-lite","quinn","rustls",- "rustls-pemfile 2.1.2",+ "rustls-pemfile 2.1.3","rustls-pki-types","serde","serde_json",@@ -4253,9 +4260,9 @@ dependencies = [[[package]]name = "rgb"-version = "0.8.45"+version = "0.8.48"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ade4539f42266ded9e755c605bdddf546242b2c961b03b06a7375260788a0523"+checksum = "0f86ae463694029097b846d8f99fd5536740602ae00022c0c50c5600720b2f71"dependencies = ["bytemuck",]@@ -4415,6 +4422,12 @@ version = "1.1.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"+[[package]]+name = "rustc-hash"+version = "2.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "583034fd73374156e66797ed8e5b0d5690409c9226b22d87cb7f19821c05d152"+[[package]]name = "rustc_version"version = "0.4.0"@@ -4462,9 +4475,9 @@ dependencies = [[[package]]name = "rustls-pemfile"-version = "2.1.2"+version = "2.1.3"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "29993a25686778eb88d4189742cd713c9bce943bc54251a33509dc63cbacf73d"+checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425"dependencies = ["base64 0.22.1","rustls-pki-types",@@ -4472,9 +4485,9 @@ dependencies = [[[package]]name = "rustls-pki-types"-version = "1.7.0"+version = "1.8.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d"+checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0"[[package]]name = "rustls-webpki"@@ -4536,9 +4549,9 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"[[package]]name = "safetensors"-version = "0.4.3"+version = "0.4.4"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "8ced76b22c7fba1162f11a5a75d9d8405264b467a07ae0c9c29be119b9297db9"+checksum = "7725d4d98fa515472f43a6e2bbf956c48e06b89bb50593a040e5945160214450"dependencies = ["serde","serde_json",@@ -4687,9 +4700,9 @@ checksum = "a3f0bf26fd526d2a95683cd0f87bf103b8539e2ca1ef48ce002d67aad59aa0b4"[[package]]name = "serde"-version = "1.0.204"+version = "1.0.205"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "bc76f558e0cbb2a839d37354c575f1dc3fdc6546b5be373ba43d95f231bf7c12"+checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150"dependencies = ["serde_derive",]@@ -4707,9 +4720,9 @@ dependencies = [[[package]]name = "serde_derive"-version = "1.0.204"+version = "1.0.205"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "e0cd7e117be63d3c3678776753929474f3b04a43a080c744d6b0ae2a8c28e222"+checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1"dependencies = ["proc-macro2","quote",@@ -4718,11 +4731,12 @@ dependencies = [[[package]]name = "serde_json"-version = "1.0.120"+version = "1.0.122"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4e0d21c9a8cae1235ad58a00c11cb40d4b1e5c784f1ef2c537876ed6ffd8b7c5"+checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"dependencies = ["itoa",+ "memchr","ryu","serde",]@@ -4748,9 +4762,9 @@ dependencies = [[[package]]name = "serde_spanned"-version = "0.6.6"+version = "0.6.7"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "79e674e01f999af37c49f70a6ede167a8a60b2503e56c5599532a65baa5969a0"+checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d"dependencies = ["serde",]@@ -5008,7 +5022,7 @@ dependencies = ["robotstxt","rust-s3","rust-stemmers",- "rustc-hash",+ "rustc-hash 1.1.0","scylla","serde","serde_json",@@ -5270,7 +5284,7 @@ dependencies = ["rayon","regex","rust-stemmers",- "rustc-hash",+ "rustc-hash 1.1.0","serde","serde_json","smallvec",@@ -5302,20 +5316,21 @@ dependencies = [[[package]]name = "target-lexicon"-version = "0.12.15"+version = "0.12.16"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4873307b7c257eddcb50c9bedf158eb669578359fb28428bef438fec8e6ba7c2"+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"[[package]]name = "tempfile"-version = "3.10.1"+version = "3.12.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "85b77fafb263dd9d05cbeac119526425676db3784113aa9295c88498cbf8bff1"+checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"dependencies = ["cfg-if","fastrand",+ "once_cell","rustix",- "windows-sys 0.52.0",+ "windows-sys 0.59.0",][[package]]@@ -5504,9 +5519,9 @@ dependencies = [[[package]]name = "tokio"-version = "1.39.1"+version = "1.39.2"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "d040ac2b29ab03b09d4129c2f5bbd012a3ac2f79d38ff506a4bf8dd34b0eac8a"+checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1"dependencies = ["backtrace","bytes",@@ -5579,21 +5594,21 @@ dependencies = [[[package]]name = "toml"-version = "0.8.15"+version = "0.8.19"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "ac2caab0bf757388c6c0ae23b3293fdb463fee59434529014f85e3263b995c28"+checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"dependencies = ["serde","serde_spanned","toml_datetime",- "toml_edit 0.22.16",+ "toml_edit 0.22.20",][[package]]name = "toml_datetime"-version = "0.6.6"+version = "0.6.8"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4badfd56924ae69bcc9039335b2e017639ce3f9b001c393c1b2d1ef846ce2cbf"+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"dependencies = ["serde",]@@ -5604,7 +5619,7 @@ version = "0.19.15"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1b5bb770da30e5cbfde35a2d7b9b8a2c4b8ef89548a7a6aeab5c9a576e3e7421"dependencies = [- "indexmap 2.2.6",+ "indexmap 2.3.0","toml_datetime","winnow 0.5.40",]@@ -5615,22 +5630,22 @@ version = "0.21.1"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "6a8534fd7f78b5405e860340ad6575217ce99f38d4d5c8f2442cb5ecb50090e1"dependencies = [- "indexmap 2.2.6",+ "indexmap 2.3.0","toml_datetime","winnow 0.5.40",][[package]]name = "toml_edit"-version = "0.22.16"+version = "0.22.20"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "278f3d518e152219c994ce877758516bca5e118eaed6996192a774fb9fbf0788"+checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"dependencies = [- "indexmap 2.2.6",+ "indexmap 2.3.0","serde","serde_spanned","toml_datetime",- "winnow 0.6.15",+ "winnow 0.6.18",][[package]]@@ -5920,7 +5935,7 @@ version = "4.2.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "c5afb1a60e207dca502682537fefcfd9921e71d0b83e9576060f09abc6efab23"dependencies = [- "indexmap 2.2.6",+ "indexmap 2.3.0","serde","serde_json","utoipa-gen",@@ -6007,9 +6022,9 @@ checksum = "852e951cb7832cb45cb1169900d19760cfa39b82bc0ea9c0e5a14ae88411c98b"[[package]]name = "version_check"-version = "0.9.4"+version = "0.9.5"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f"+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"[[package]]name = "virtue"@@ -6193,11 +6208,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"[[package]]name = "winapi-util"-version = "0.1.8"+version = "0.1.9"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "4d4cc384e1e73b93bafa6fb4f1df8c41695c8a91cf9c4c64358067d15a7b6c6b"+checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb"dependencies = [- "windows-sys 0.52.0",+ "windows-sys 0.59.0",][[package]]@@ -6233,6 +6248,15 @@ dependencies = ["windows-targets 0.52.6",]+[[package]]+name = "windows-sys"+version = "0.59.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"+dependencies = [+ "windows-targets 0.52.6",+]+[[package]]name = "windows-targets"version = "0.48.5"@@ -6365,9 +6389,9 @@ dependencies = [[[package]]name = "winnow"-version = "0.6.15"+version = "0.6.18"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "557404e450152cd6795bb558bca69e43c585055f4606e3bcae5894fc6dac9ba0"+checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"dependencies = ["memchr",]@@ -6414,9 +6438,9 @@ dependencies = [[[package]]name = "xxhash-rust"-version = "0.8.11"+version = "0.8.12"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "63658493314859b4dfdf3fb8c1defd61587839def09582db50b8a4e93afca6bb"+checksum = "6a5cbf750400958819fb6178eaa83bee5cd9c29a26a40cc241df8c70fdd46984"[[package]]name = "yansi"@@ -6463,6 +6487,7 @@ version = "0.7.35"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"dependencies = [+ "byteorder 1.5.0","zerocopy-derive",]@@ -6545,8 +6570,8 @@ dependencies = ["crossbeam-utils","displaydoc","flate2",- "indexmap 2.2.6",- "num_enum 0.7.2",+ "indexmap 2.3.0",+ "num_enum 0.7.3","thiserror",]@@ -6574,7 +6599,7 @@ version = "0.13.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "fcf2b778a664581e31e389454a7072dab1647606d44f7feea22cd5abb9c9f3f9"dependencies = [- "zstd-safe 7.2.0",+ "zstd-safe 7.2.1",][[package]]@@ -6589,18 +6614,18 @@ dependencies = [[[package]]name = "zstd-safe"-version = "7.2.0"+version = "7.2.1"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "fa556e971e7b568dc775c136fc9de8c779b1c2fc3a63defaafadffdbd3181afa"+checksum = "54a3ab4db68cea366acc5c897c7b4d4d1b8994a9cd6e6f841f8964566a419059"dependencies = ["zstd-sys",][[package]]name = "zstd-sys"-version = "2.0.12+zstd.1.5.6"+version = "2.0.13+zstd.1.5.6"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "0a4e40c320c3cb459d9a9ff6de98cff88f4751ee9275d140e2be94a2b74e4c13"+checksum = "38ff0f21cfee8f97d94cef41359e0c89aa6113028ab0291aa8ca0038995a95aa"dependencies = ["cc","pkg-config",diff --git a/Cargo.toml b/Cargo.tomlindex 9373d7dc..423bb260 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -13,6 +13,7 @@ members = ["crates/lending-iter","crates/robotstxt","crates/tantivy",+ "crates/leechy","fuzz",]resolver = "2"diff --git a/crates/kuchiki/src/iter.rs b/crates/kuchiki/src/iter.rsindex fa838386..02405e8a 100644--- a/crates/kuchiki/src/iter.rs+++ b/crates/kuchiki/src/iter.rs@@ -4,6 +4,8 @@ use std::borrow::Borrow;use std::cell::RefCell;use std::iter::Rev;+use selectors::Element;+use crate::node_data_ref::NodeDataRef;use crate::select::Selectors;use crate::tree::{ElementData, NodeRef};@@ -170,6 +172,12 @@ impl NodeRef {self.inclusive_descendants().select(selectors)}+ /// Return the node that matches the given xpath expression.+ #[inline]+ pub fn select_xpath(&self, xpath: &str) -> XPathSelect<Elements<Descendants>> {+ XPathSelect::new(self.inclusive_descendants().elements(), xpath)+ }+/// Return the first inclusive descendants element that match the given selector list.#[inline]#[must_use]@@ -179,6 +187,69 @@ impl NodeRef {}}+/// An iterator of nodes that match a given xpath selector.+pub struct XPathSelect<I> {+ iter: I,+ xpath: Vec<crate::xpath::Expr>,+}++impl<I> XPathSelect<I> {+ /// Create a new `XPathSelect` iterator.+ pub fn new(iter: I, xpath: &str) -> Self {+ let xpath = crate::xpath::Expr::parse(xpath);+ Self { iter, xpath }+ }+}++impl<I> Iterator for XPathSelect<I>+where+ I: Iterator<Item = NodeDataRef<ElementData>>,+{+ type Item = NodeDataRef<ElementData>;++ fn next(&mut self) -> Option<Self::Item> {+ self.iter.by_ref().find(|elem| {+ let mut elem = Some(elem.clone());+ let mut expr_index = self.xpath.len();++ while expr_index > 0 {+ let expr = &self.xpath[expr_index - 1];++ elem = match elem {+ Some(elem) => {+ if !expr.matches(&elem) {+ return false;+ }++ elem.parent_element()+ }++ None => {+ return matches!(+ expr,+ crate::xpath::Expr::Root | crate::xpath::Expr::Wildcard+ );+ }+ };++ if matches!(expr, crate::xpath::Expr::Wildcard)+ && expr_index > 1+ && elem+ .as_ref()+ .map(|elem| !self.xpath[expr_index - 2].matches(elem))+ .unwrap_or(false)+ {+ expr_index -= 0;+ } else {+ expr_index -= 1;+ }+ }++ true+ })+ }+}+#[derive(Debug, Clone)]struct State<T> {next: T,diff --git a/crates/kuchiki/src/lib.rs b/crates/kuchiki/src/lib.rsindex bbe14e5f..897d0049 100644--- a/crates/kuchiki/src/lib.rs+++ b/crates/kuchiki/src/lib.rs@@ -18,6 +18,7 @@ mod node_data_ref;mod parser;mod select;mod serializer;+mod xpath;#[cfg(test)]mod tests;diff --git a/crates/kuchiki/src/tests.rs b/crates/kuchiki/src/tests.rsindex 83edbb6c..214320d6 100644--- a/crates/kuchiki/src/tests.rs+++ b/crates/kuchiki/src/tests.rs@@ -189,3 +189,83 @@ fn specificity() {assert!(specificities[0] > specificities[2]);assert!(specificities[1] > specificities[2]);}++#[test]+fn select_xpath() {+ let html = r"+<title>Test case</title>+<p class=foo>Foo</p>+<p>Bar</p>+<p class=foo>Foo</p>+";++ let document = parse_html().one(html);+ let nodes: Vec<_> = document.select_xpath("/html/body/p").collect();++ assert_eq!(nodes.len(), 3);++ let node = &nodes[0];+ assert_eq!(node.name.local.as_ref(), "p");+ assert_eq!(node.attributes.borrow().get("class"), Some("foo"));++ let node = &nodes[1];+ assert_eq!(node.name.local.as_ref(), "p");+ assert_eq!(node.attributes.borrow().get("class"), None);++ let node = &nodes[2];+ assert_eq!(node.name.local.as_ref(), "p");+ assert_eq!(node.attributes.borrow().get("class"), Some("foo"));++ let nodes: Vec<_> = document.select_xpath("/html/body/p[2]").collect();++ assert_eq!(nodes.len(), 1);++ let node = &nodes[0];+ assert_eq!(node.name.local.as_ref(), "p");+ assert_eq!(node.attributes.borrow().get("class"), None);++ let nodes: Vec<_> = document.select_xpath("/html/head/title").collect();+ assert_eq!(nodes.len(), 1);+}++#[test]+fn xpath_wildcard() {+ let html = r"+<title>Test case</title>+";++ let document = parse_html().one(html);++ let nodes: Vec<_> = document.select_xpath("/title").collect();+ assert_eq!(nodes.len(), 0);++ let nodes: Vec<_> = document.select_xpath("./title").collect();+ assert_eq!(nodes.len(), 1);++ let nodes: Vec<_> = document.select_xpath("/./title").collect();+ assert_eq!(nodes.len(), 1);+}++#[test]+fn xpath_contains() {+ let html = r#"+<p class="foo bar">Test case</p>+"#;++ let document = parse_html().one(html);++ let nodes: Vec<_> = document+ .select_xpath("./p[contains(@class, 'foo')]")+ .collect();+ assert_eq!(nodes.len(), 1);++ let nodes: Vec<_> = document+ .select_xpath("./p[contains(@class, 'bar')]")+ .collect();+ assert_eq!(nodes.len(), 1);++ let nodes: Vec<_> = document+ .select_xpath("./p[contains(@class, 'baz')]")+ .collect();+ assert_eq!(nodes.len(), 0);+}diff --git a/crates/kuchiki/src/xpath.rs b/crates/kuchiki/src/xpath.rsnew file mode 100644index 00000000..227ba3c5--- /dev/null+++ b/crates/kuchiki/src/xpath.rs@@ -0,0 +1,297 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use selectors::Element;++use crate::{iter::NodeIterator, ElementData, NodeDataRef};++#[derive(Debug, PartialEq)]+pub enum Predicate {+ Nth(usize),+ Contains { key: ContainsKey, value: String },+}++#[derive(Debug, PartialEq)]+pub enum ContainsKey {+ Text,+ Attr(String),+}++impl Predicate {+ pub fn parse(s: &str) -> Option<Self> {+ if let Ok(num) = s.parse::<usize>() {+ if num > 0 {+ return Some(Predicate::Nth(num - 1));+ }+ }++ if s.starts_with("contains(text(),") {+ let value = s+ .trim_start_matches("contains(text(),")+ .trim_end_matches(')');+ return Some(Predicate::Contains {+ key: ContainsKey::Text,+ value: value.to_string(),+ });+ }++ if s.starts_with("contains(@") {+ let attr_val = s.trim_start_matches("contains(@").trim_end_matches(')');++ let mut parts = attr_val.split(',');++ if let (Some(attr), Some(value)) = (parts.next(), parts.next()) {+ return Some(Predicate::Contains {+ key: ContainsKey::Attr(attr.to_string()),+ value: value+ .trim()+ .trim_start_matches(|c| matches!(c, '\'' | '"'))+ .trim_end_matches(|c| matches!(c, '\'' | '"'))+ .to_string(),+ });+ }+ }++ None+ }++ pub fn matches(&self, elem: &NodeDataRef<ElementData>) -> bool {+ match self {+ Predicate::Nth(nth) => {+ let parent = elem.parent_element().unwrap();+ parent+ .as_node()+ .children()+ .elements()+ .nth(*nth)+ .map(|child| &child == elem)+ .unwrap_or(false)+ }++ Predicate::Contains { key, value } => match key {+ ContainsKey::Text => elem.text_contents().contains(value),+ ContainsKey::Attr(attr) => elem+ .attributes+ .borrow()+ .get(attr.as_str())+ .map_or(false, |v| v.split_ascii_whitespace().any(|v| v == value)),+ },+ }+ }+}++#[derive(Debug, PartialEq)]+pub enum Expr {+ Root,+ Wildcard,+ Child {+ name: String,+ pred: Option<Predicate>,+ },+}++impl Expr {+ pub fn parse(s: &str) -> Vec<Self> {+ let mut tokens = Vec::new();+ let mut current = String::new();+ let mut in_brackets = false;++ for c in s.chars() {+ match c {+ '/' if !in_brackets => {+ if !current.is_empty() {+ tokens.push(current);+ current = String::new();+ }+ tokens.push("/".to_string());+ }+ '[' => {+ in_brackets = true;+ tokens.push(current);+ current = String::new();+ current.push('[');+ }+ ']' => {+ in_brackets = false;+ current.push(']');+ tokens.push(current);+ current = String::new();+ }+ _ => current.push(c),+ }+ }++ if !current.is_empty() {+ tokens.push(current);+ }++ let mut result = Vec::new();+ let mut is_root = true;++ let mut i = 0;+ while i < tokens.len() {+ match tokens[i].as_str() {+ "/" => {+ if is_root {+ result.push(Expr::Root);+ is_root = false;+ }+ }+ _ => {+ let name = tokens[i].to_string();++ if name == "." || name == "*" {+ result.push(Expr::Wildcard);+ i += 1;+ is_root = false;+ continue;+ }++ let mut pred = None;++ if i + 1 < tokens.len()+ && tokens[i + 1].starts_with('[')+ && tokens[i + 1].ends_with(']')+ {+ if let Some(p) =+ Predicate::parse(&tokens[i + 1][1..tokens[i + 1].len() - 1])+ {+ pred = Some(p);+ i += 1;+ }+ }++ result.push(Expr::Child {+ name: name.trim_start_matches('/').to_string(),+ pred,+ });+ }+ }+ i += 1;+ }++ result+ }++ pub fn matches(&self, elem: &NodeDataRef<ElementData>) -> bool {+ match self {+ Expr::Root => elem.parent_element().is_none(),+ Expr::Child { name, pred } => {+ if elem.name.local.as_ref() != name {+ return false;+ }++ if let Some(pred) = pred {+ pred.matches(elem)+ } else {+ true+ }+ }+ Expr::Wildcard => true,+ }+ }+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn test_expr_parse() {+ assert_eq!(Expr::parse("/"), vec![Expr::Root]);+ assert_eq!(+ Expr::parse("/html"),+ vec![+ Expr::Root,+ Expr::Child {+ name: "html".to_string(),+ pred: None+ }+ ]+ );+ assert_eq!(+ Expr::parse("/html[1]"),+ vec![+ Expr::Root,+ Expr::Child {+ name: "html".to_string(),+ pred: Some(Predicate::Nth(0))+ }+ ]+ );+ assert_eq!(+ Expr::parse("/html[1]/body"),+ vec![+ Expr::Root,+ Expr::Child {+ name: "html".to_string(),+ pred: Some(Predicate::Nth(0))+ },+ Expr::Child {+ name: "body".to_string(),+ pred: None+ }+ ]+ );+ assert_eq!(+ Expr::parse("/html[1]/body[1]"),+ vec![+ Expr::Root,+ Expr::Child {+ name: "html".to_string(),+ pred: Some(Predicate::Nth(0))+ },+ Expr::Child {+ name: "body".to_string(),+ pred: Some(Predicate::Nth(0))+ }+ ]+ );+ assert_eq!(+ Expr::parse("/html[1]/body[1]"),+ vec![+ Expr::Root,+ Expr::Child {+ name: "html".to_string(),+ pred: Some(Predicate::Nth(0))+ },+ Expr::Child {+ name: "body".to_string(),+ pred: Some(Predicate::Nth(0))+ },+ ]+ );++ assert_eq!(+ Expr::parse("/html/body/div[4]"),+ vec![+ Expr::Root,+ Expr::Child {+ name: "html".to_string(),+ pred: None+ },+ Expr::Child {+ name: "body".to_string(),+ pred: None+ },+ Expr::Child {+ name: "div".to_string(),+ pred: Some(Predicate::Nth(3))+ }+ ]+ );+ }+}diff --git a/crates/leechy-py/.gitignore b/crates/leechy-py/.gitignorenew file mode 100644index 00000000..c8f04429--- /dev/null+++ b/crates/leechy-py/.gitignore@@ -0,0 +1,72 @@+/target++# Byte-compiled / optimized / DLL files+__pycache__/+.pytest_cache/+*.py[cod]++# C extensions+*.so++# Distribution / packaging+.Python+.venv/+env/+bin/+build/+develop-eggs/+dist/+eggs/+lib/+lib64/+parts/+sdist/+var/+include/+man/+venv/+*.egg-info/+.installed.cfg+*.egg++# Installer logs+pip-log.txt+pip-delete-this-directory.txt+pip-selfcheck.json++# Unit test / coverage reports+htmlcov/+.tox/+.coverage+.cache+nosetests.xml+coverage.xml++# Translations+*.mo++# Mr Developer+.mr.developer.cfg+.project+.pydevproject++# Rope+.ropeproject++# Django stuff:+*.log+*.pot++.DS_Store++# Sphinx documentation+docs/_build/++# PyCharm+.idea/++# VSCode+.vscode/++# Pyenv+.python-versiondiff --git a/crates/leechy-py/Cargo.lock b/crates/leechy-py/Cargo.locknew file mode 100644index 00000000..467ffc1d--- /dev/null+++ b/crates/leechy-py/Cargo.lock@@ -0,0 +1,1823 @@+# This file is automatically @generated by Cargo.+# It is not intended for manual editing.+version = 3++[[package]]+name = "addr2line"+version = "0.22.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6e4503c46a5c0c7844e948c9a4d6acd9f50cccb4de1c48eb9e291ea17470c678"+dependencies = [+ "gimli",+]++[[package]]+name = "adler"+version = "1.0.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"++[[package]]+name = "anyhow"+version = "1.0.86"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da"+dependencies = [+ "backtrace",+]++[[package]]+name = "autocfg"+version = "1.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0"++[[package]]+name = "backtrace"+version = "0.3.73"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5cc23269a4f8976d0a4d2e7109211a419fe30e8d88d677cd60b6bc79c5732e0a"+dependencies = [+ "addr2line",+ "cc",+ "cfg-if",+ "libc",+ "miniz_oxide",+ "object",+ "rustc-demangle",+]++[[package]]+name = "base64"+version = "0.21.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"++[[package]]+name = "bitflags"+version = "1.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"++[[package]]+name = "bitflags"+version = "2.6.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de"++[[package]]+name = "bumpalo"+version = "3.16.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c"++[[package]]+name = "byteorder"+version = "1.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"++[[package]]+name = "bytes"+version = "1.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8318a53db07bb3f8dca91a600466bdb3f2eaadeedfdbcf02e1accbad9271ba50"++[[package]]+name = "cc"+version = "1.1.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "504bdec147f2cc13c8b57ed9401fd8a147cc66b67ad5cb241394244f2c947549"++[[package]]+name = "cfg-if"+version = "1.0.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"++[[package]]+name = "core-foundation"+version = "0.9.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f"+dependencies = [+ "core-foundation-sys",+ "libc",+]++[[package]]+name = "core-foundation-sys"+version = "0.8.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "06ea2b9bc92be3c2baa9334a323ebca2d6f074ff852cd1d7b11064035cd3868f"++[[package]]+name = "cssparser"+version = "0.31.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5b3df4f93e5fbbe73ec01ec8d3f68bba73107993a5b1e7519273c32db9b0d5be"+dependencies = [+ "cssparser-macros",+ "dtoa-short",+ "itoa",+ "phf 0.11.2",+ "smallvec",+]++[[package]]+name = "cssparser-macros"+version = "0.6.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "13b588ba4ac1a99f7f2964d24b3d896ddc6bf847ee3855dbd4366f058cfcd331"+dependencies = [+ "quote",+ "syn 2.0.72",+]++[[package]]+name = "derive_more"+version = "0.99.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5f33878137e4dafd7fa914ad4e259e18a4e8e532b9617a2d0150262bf53abfce"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.72",+]++[[package]]+name = "dtoa"+version = "1.0.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653"++[[package]]+name = "dtoa-short"+version = "0.3.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "cd1511a7b6a56299bd043a9c167a6d2bfb37bf84a6dfceaba651168adfb43c87"+dependencies = [+ "dtoa",+]++[[package]]+name = "encoding_rs"+version = "0.8.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"+dependencies = [+ "cfg-if",+]++[[package]]+name = "equivalent"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5"++[[package]]+name = "errno"+version = "0.3.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba"+dependencies = [+ "libc",+ "windows-sys 0.52.0",+]++[[package]]+name = "fastrand"+version = "2.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a"++[[package]]+name = "fnv"+version = "1.0.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"++[[package]]+name = "foreign-types"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1"+dependencies = [+ "foreign-types-shared",+]++[[package]]+name = "foreign-types-shared"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"++[[package]]+name = "form_urlencoded"+version = "1.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456"+dependencies = [+ "percent-encoding",+]++[[package]]+name = "futf"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843"+dependencies = [+ "mac",+ "new_debug_unreachable",+]++[[package]]+name = "futures-channel"+version = "0.3.30"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78"+dependencies = [+ "futures-core",+]++[[package]]+name = "futures-core"+version = "0.3.30"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d"++[[package]]+name = "futures-io"+version = "0.3.30"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1"++[[package]]+name = "futures-macro"+version = "0.3.30"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.72",+]++[[package]]+name = "futures-sink"+version = "0.3.30"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5"++[[package]]+name = "futures-task"+version = "0.3.30"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004"++[[package]]+name = "futures-util"+version = "0.3.30"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48"+dependencies = [+ "futures-core",+ "futures-io",+ "futures-macro",+ "futures-sink",+ "futures-task",+ "memchr",+ "pin-project-lite",+ "pin-utils",+ "slab",+]++[[package]]+name = "fxhash"+version = "0.2.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"+dependencies = [+ "byteorder",+]++[[package]]+name = "getrandom"+version = "0.2.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"+dependencies = [+ "cfg-if",+ "libc",+ "wasi",+]++[[package]]+name = "gimli"+version = "0.29.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd"++[[package]]+name = "h2"+version = "0.3.26"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "81fe527a889e1532da5c525686d96d4c2e74cdd345badf8dfef9f6b39dd5f5e8"+dependencies = [+ "bytes",+ "fnv",+ "futures-core",+ "futures-sink",+ "futures-util",+ "http",+ "indexmap 2.3.0",+ "slab",+ "tokio",+ "tokio-util",+ "tracing",+]++[[package]]+name = "hashbrown"+version = "0.12.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"++[[package]]+name = "hashbrown"+version = "0.14.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"++[[package]]+name = "heck"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"++[[package]]+name = "hermit-abi"+version = "0.3.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024"++[[package]]+name = "html5ever"+version = "0.26.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7"+dependencies = [+ "log",+ "mac",+ "markup5ever",+ "proc-macro2",+ "quote",+ "syn 1.0.109",+]++[[package]]+name = "http"+version = "0.2.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1"+dependencies = [+ "bytes",+ "fnv",+ "itoa",+]++[[package]]+name = "http-body"+version = "0.4.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7ceab25649e9960c0311ea418d17bee82c0dcec1bd053b5f9a66e265a693bed2"+dependencies = [+ "bytes",+ "http",+ "pin-project-lite",+]++[[package]]+name = "httparse"+version = "1.9.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9"++[[package]]+name = "httpdate"+version = "1.0.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"++[[package]]+name = "hyper"+version = "0.14.30"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a152ddd61dfaec7273fe8419ab357f33aee0d914c5f4efbf0d96fa749eea5ec9"+dependencies = [+ "bytes",+ "futures-channel",+ "futures-core",+ "futures-util",+ "h2",+ "http",+ "http-body",+ "httparse",+ "httpdate",+ "itoa",+ "pin-project-lite",+ "socket2",+ "tokio",+ "tower-service",+ "tracing",+ "want",+]++[[package]]+name = "hyper-tls"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d6183ddfa99b85da61a140bea0efc93fdf56ceaa041b37d553518030827f9905"+dependencies = [+ "bytes",+ "hyper",+ "native-tls",+ "tokio",+ "tokio-native-tls",+]++[[package]]+name = "idna"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "634d9b1461af396cad843f47fdba5597a4f9e6ddd4bfb6ff5d85028c25cb12f6"+dependencies = [+ "unicode-bidi",+ "unicode-normalization",+]++[[package]]+name = "indexmap"+version = "1.9.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"+dependencies = [+ "autocfg",+ "hashbrown 0.12.3",+]++[[package]]+name = "indexmap"+version = "2.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "de3fc2e30ba82dd1b3911c8de1ffc143c74a914a14e99514d7637e3099df5ea0"+dependencies = [+ "equivalent",+ "hashbrown 0.14.5",+]++[[package]]+name = "indoc"+version = "2.0.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b248f5224d1d606005e02c97f5aa4e88eeb230488bcc03bc9ca4d7991399f2b5"++[[package]]+name = "ipnet"+version = "2.9.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f518f335dce6725a761382244631d86cf0ccb2863413590b31338feb467f9c3"++[[package]]+name = "itoa"+version = "1.0.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b"++[[package]]+name = "js-sys"+version = "0.3.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "29c15563dc2726973df627357ce0c9ddddbea194836909d655df6a75d2cf296d"+dependencies = [+ "wasm-bindgen",+]++[[package]]+name = "kuchiki"+version = "0.8.1"+dependencies = [+ "cssparser",+ "html5ever",+ "indexmap 1.9.3",+ "matches",+ "selectors",+ "thiserror",+]++[[package]]+name = "leechy"+version = "0.1.0"+dependencies = [+ "anyhow",+ "kuchiki",+ "rand",+ "reqwest",+ "serde",+ "toml",+]++[[package]]+name = "leechy"+version = "0.1.0+py"+dependencies = [+ "leechy 0.1.0",+ "pyo3",+ "tokio",+]++[[package]]+name = "libc"+version = "0.2.155"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c"++[[package]]+name = "linux-raw-sys"+version = "0.4.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89"++[[package]]+name = "lock_api"+version = "0.4.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17"+dependencies = [+ "autocfg",+ "scopeguard",+]++[[package]]+name = "log"+version = "0.4.22"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a7a70ba024b9dc04c27ea2f0c0548feb474ec5c54bba33a7f72f873a39d07b24"++[[package]]+name = "mac"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"++[[package]]+name = "markup5ever"+version = "0.11.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016"+dependencies = [+ "log",+ "phf 0.10.1",+ "phf_codegen",+ "string_cache",+ "string_cache_codegen",+ "tendril",+]++[[package]]+name = "matches"+version = "0.1.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2532096657941c2fea9c289d370a250971c689d4f143798ff67113ec042024a5"++[[package]]+name = "memchr"+version = "2.7.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3"++[[package]]+name = "memoffset"+version = "0.9.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"+dependencies = [+ "autocfg",+]++[[package]]+name = "mime"+version = "0.3.17"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"++[[package]]+name = "miniz_oxide"+version = "0.7.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b8a240ddb74feaf34a79a7add65a741f3167852fba007066dcac1ca548d89c08"+dependencies = [+ "adler",+]++[[package]]+name = "mio"+version = "1.0.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4569e456d394deccd22ce1c1913e6ea0e54519f577285001215d33557431afe4"+dependencies = [+ "hermit-abi",+ "libc",+ "wasi",+ "windows-sys 0.52.0",+]++[[package]]+name = "native-tls"+version = "0.2.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a8614eb2c83d59d1c8cc974dd3f920198647674a0a035e1af1fa58707e317466"+dependencies = [+ "libc",+ "log",+ "openssl",+ "openssl-probe",+ "openssl-sys",+ "schannel",+ "security-framework",+ "security-framework-sys",+ "tempfile",+]++[[package]]+name = "new_debug_unreachable"+version = "1.0.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086"++[[package]]+name = "object"+version = "0.36.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "27b64972346851a39438c60b341ebc01bba47464ae329e55cf343eb93964efd9"+dependencies = [+ "memchr",+]++[[package]]+name = "once_cell"+version = "1.19.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92"++[[package]]+name = "openssl"+version = "0.10.66"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9529f4786b70a3e8c61e11179af17ab6188ad8d0ded78c5529441ed39d4bd9c1"+dependencies = [+ "bitflags 2.6.0",+ "cfg-if",+ "foreign-types",+ "libc",+ "once_cell",+ "openssl-macros",+ "openssl-sys",+]++[[package]]+name = "openssl-macros"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.72",+]++[[package]]+name = "openssl-probe"+version = "0.1.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"++[[package]]+name = "openssl-sys"+version = "0.9.103"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "7f9e8deee91df40a943c71b917e5874b951d32a802526c85721ce3b776c929d6"+dependencies = [+ "cc",+ "libc",+ "pkg-config",+ "vcpkg",+]++[[package]]+name = "parking_lot"+version = "0.12.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27"+dependencies = [+ "lock_api",+ "parking_lot_core",+]++[[package]]+name = "parking_lot_core"+version = "0.9.10"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8"+dependencies = [+ "cfg-if",+ "libc",+ "redox_syscall",+ "smallvec",+ "windows-targets 0.52.6",+]++[[package]]+name = "percent-encoding"+version = "2.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e"++[[package]]+name = "phf"+version = "0.10.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259"+dependencies = [+ "phf_shared 0.10.0",+]++[[package]]+name = "phf"+version = "0.11.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc"+dependencies = [+ "phf_macros",+ "phf_shared 0.11.2",+]++[[package]]+name = "phf_codegen"+version = "0.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd"+dependencies = [+ "phf_generator 0.10.0",+ "phf_shared 0.10.0",+]++[[package]]+name = "phf_generator"+version = "0.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6"+dependencies = [+ "phf_shared 0.10.0",+ "rand",+]++[[package]]+name = "phf_generator"+version = "0.11.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0"+dependencies = [+ "phf_shared 0.11.2",+ "rand",+]++[[package]]+name = "phf_macros"+version = "0.11.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b"+dependencies = [+ "phf_generator 0.11.2",+ "phf_shared 0.11.2",+ "proc-macro2",+ "quote",+ "syn 2.0.72",+]++[[package]]+name = "phf_shared"+version = "0.10.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096"+dependencies = [+ "siphasher",+]++[[package]]+name = "phf_shared"+version = "0.11.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b"+dependencies = [+ "siphasher",+]++[[package]]+name = "pin-project-lite"+version = "0.2.14"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02"++[[package]]+name = "pin-utils"+version = "0.1.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"++[[package]]+name = "pkg-config"+version = "0.3.30"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d231b230927b5e4ad203db57bbcbee2802f6bce620b1e4a9024a07d94e2907ec"++[[package]]+name = "portable-atomic"+version = "1.7.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "da544ee218f0d287a911e9c99a39a8c9bc8fcad3cb8db5959940044ecfc67265"++[[package]]+name = "ppv-lite86"+version = "0.2.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "77957b295656769bb8ad2b6a6b09d897d94f05c41b069aede1fcdaa675eaea04"+dependencies = [+ "zerocopy",+]++[[package]]+name = "precomputed-hash"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c"++[[package]]+name = "proc-macro2"+version = "1.0.86"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77"+dependencies = [+ "unicode-ident",+]++[[package]]+name = "pyo3"+version = "0.22.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "831e8e819a138c36e212f3af3fd9eeffed6bf1510a805af35b0edee5ffa59433"+dependencies = [+ "cfg-if",+ "indoc",+ "libc",+ "memoffset",+ "once_cell",+ "portable-atomic",+ "pyo3-build-config",+ "pyo3-ffi",+ "pyo3-macros",+ "unindent",+]++[[package]]+name = "pyo3-build-config"+version = "0.22.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e8730e591b14492a8945cdff32f089250b05f5accecf74aeddf9e8272ce1fa8"+dependencies = [+ "once_cell",+ "target-lexicon",+]++[[package]]+name = "pyo3-ffi"+version = "0.22.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "5e97e919d2df92eb88ca80a037969f44e5e70356559654962cbb3316d00300c6"+dependencies = [+ "libc",+ "pyo3-build-config",+]++[[package]]+name = "pyo3-macros"+version = "0.22.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "eb57983022ad41f9e683a599f2fd13c3664d7063a3ac5714cae4b7bee7d3f206"+dependencies = [+ "proc-macro2",+ "pyo3-macros-backend",+ "quote",+ "syn 2.0.72",+]++[[package]]+name = "pyo3-macros-backend"+version = "0.22.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec480c0c51ddec81019531705acac51bcdbeae563557c982aa8263bb96880372"+dependencies = [+ "heck",+ "proc-macro2",+ "pyo3-build-config",+ "quote",+ "syn 2.0.72",+]++[[package]]+name = "quote"+version = "1.0.36"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"+dependencies = [+ "proc-macro2",+]++[[package]]+name = "rand"+version = "0.8.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"+dependencies = [+ "libc",+ "rand_chacha",+ "rand_core",+]++[[package]]+name = "rand_chacha"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"+dependencies = [+ "ppv-lite86",+ "rand_core",+]++[[package]]+name = "rand_core"+version = "0.6.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"+dependencies = [+ "getrandom",+]++[[package]]+name = "redox_syscall"+version = "0.5.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2a908a6e00f1fdd0dfd9c0eb08ce85126f6d8bbda50017e74bc4a4b7d4a926a4"+dependencies = [+ "bitflags 2.6.0",+]++[[package]]+name = "reqwest"+version = "0.11.27"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dd67538700a17451e7cba03ac727fb961abb7607553461627b97de0b89cf4a62"+dependencies = [+ "base64",+ "bytes",+ "encoding_rs",+ "futures-core",+ "futures-util",+ "h2",+ "http",+ "http-body",+ "hyper",+ "hyper-tls",+ "ipnet",+ "js-sys",+ "log",+ "mime",+ "native-tls",+ "once_cell",+ "percent-encoding",+ "pin-project-lite",+ "rustls-pemfile",+ "serde",+ "serde_json",+ "serde_urlencoded",+ "sync_wrapper",+ "system-configuration",+ "tokio",+ "tokio-native-tls",+ "tokio-util",+ "tower-service",+ "url",+ "wasm-bindgen",+ "wasm-bindgen-futures",+ "wasm-streams",+ "web-sys",+ "winreg",+]++[[package]]+name = "rustc-demangle"+version = "0.1.24"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f"++[[package]]+name = "rustix"+version = "0.38.34"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f"+dependencies = [+ "bitflags 2.6.0",+ "errno",+ "libc",+ "linux-raw-sys",+ "windows-sys 0.52.0",+]++[[package]]+name = "rustls-pemfile"+version = "1.0.4"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c"+dependencies = [+ "base64",+]++[[package]]+name = "ryu"+version = "1.0.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f"++[[package]]+name = "schannel"+version = "0.1.23"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fbc91545643bcf3a0bbb6569265615222618bdf33ce4ffbbd13c4bbd4c093534"+dependencies = [+ "windows-sys 0.52.0",+]++[[package]]+name = "scopeguard"+version = "1.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"++[[package]]+name = "security-framework"+version = "2.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02"+dependencies = [+ "bitflags 2.6.0",+ "core-foundation",+ "core-foundation-sys",+ "libc",+ "security-framework-sys",+]++[[package]]+name = "security-framework-sys"+version = "2.11.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "75da29fe9b9b08fe9d6b22b5b4bcbc75d8db3aa31e639aa56bb62e9d46bfceaf"+dependencies = [+ "core-foundation-sys",+ "libc",+]++[[package]]+name = "selectors"+version = "0.25.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4eb30575f3638fc8f6815f448d50cb1a2e255b0897985c8c59f4d37b72a07b06"+dependencies = [+ "bitflags 2.6.0",+ "cssparser",+ "derive_more",+ "fxhash",+ "log",+ "new_debug_unreachable",+ "phf 0.10.1",+ "phf_codegen",+ "precomputed-hash",+ "servo_arc",+ "smallvec",+]++[[package]]+name = "serde"+version = "1.0.205"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e33aedb1a7135da52b7c21791455563facbbcc43d0f0f66165b42c21b3dfb150"+dependencies = [+ "serde_derive",+]++[[package]]+name = "serde_derive"+version = "1.0.205"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "692d6f5ac90220161d6774db30c662202721e64aed9058d2c394f451261420c1"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.72",+]++[[package]]+name = "serde_json"+version = "1.0.122"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "784b6203951c57ff748476b126ccb5e8e2959a5c19e5c617ab1956be3dbc68da"+dependencies = [+ "itoa",+ "memchr",+ "ryu",+ "serde",+]++[[package]]+name = "serde_spanned"+version = "0.6.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "eb5b1b31579f3811bf615c144393417496f152e12ac8b7663bf664f4a815306d"+dependencies = [+ "serde",+]++[[package]]+name = "serde_urlencoded"+version = "0.7.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"+dependencies = [+ "form_urlencoded",+ "itoa",+ "ryu",+ "serde",+]++[[package]]+name = "servo_arc"+version = "0.3.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d036d71a959e00c77a63538b90a6c2390969f9772b096ea837205c6bd0491a44"+dependencies = [+ "stable_deref_trait",+]++[[package]]+name = "signal-hook-registry"+version = "1.4.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1"+dependencies = [+ "libc",+]++[[package]]+name = "siphasher"+version = "0.3.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d"++[[package]]+name = "slab"+version = "0.4.9"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67"+dependencies = [+ "autocfg",+]++[[package]]+name = "smallvec"+version = "1.13.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67"++[[package]]+name = "socket2"+version = "0.5.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c"+dependencies = [+ "libc",+ "windows-sys 0.52.0",+]++[[package]]+name = "stable_deref_trait"+version = "1.2.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"++[[package]]+name = "string_cache"+version = "0.8.7"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b"+dependencies = [+ "new_debug_unreachable",+ "once_cell",+ "parking_lot",+ "phf_shared 0.10.0",+ "precomputed-hash",+ "serde",+]++[[package]]+name = "string_cache_codegen"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988"+dependencies = [+ "phf_generator 0.10.0",+ "phf_shared 0.10.0",+ "proc-macro2",+ "quote",+]++[[package]]+name = "syn"+version = "1.0.109"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"+dependencies = [+ "proc-macro2",+ "quote",+ "unicode-ident",+]++[[package]]+name = "syn"+version = "2.0.72"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc4b9b9bf2add8093d3f2c0204471e951b2285580335de42f9d2534f3ae7a8af"+dependencies = [+ "proc-macro2",+ "quote",+ "unicode-ident",+]++[[package]]+name = "sync_wrapper"+version = "0.1.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160"++[[package]]+name = "system-configuration"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ba3a3adc5c275d719af8cb4272ea1c4a6d668a777f37e115f6d11ddbc1c8e0e7"+dependencies = [+ "bitflags 1.3.2",+ "core-foundation",+ "system-configuration-sys",+]++[[package]]+name = "system-configuration-sys"+version = "0.5.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a75fb188eb626b924683e3b95e3a48e63551fcfb51949de2f06a9d91dbee93c9"+dependencies = [+ "core-foundation-sys",+ "libc",+]++[[package]]+name = "target-lexicon"+version = "0.12.16"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "61c41af27dd6d1e27b1b16b489db798443478cef1f06a660c96db617ba5de3b1"++[[package]]+name = "tempfile"+version = "3.12.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "04cbcdd0c794ebb0d4cf35e88edd2f7d2c4c3e9a5a6dab322839b321c6a87a64"+dependencies = [+ "cfg-if",+ "fastrand",+ "once_cell",+ "rustix",+ "windows-sys 0.59.0",+]++[[package]]+name = "tendril"+version = "0.4.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0"+dependencies = [+ "futf",+ "mac",+ "utf-8",+]++[[package]]+name = "thiserror"+version = "1.0.63"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c0342370b38b6a11b6cc11d6a805569958d54cfa061a29969c3b5ce2ea405724"+dependencies = [+ "thiserror-impl",+]++[[package]]+name = "thiserror-impl"+version = "1.0.63"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a4558b58466b9ad7ca0f102865eccc95938dca1a74a856f2b57b6629050da261"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.72",+]++[[package]]+name = "tinyvec"+version = "1.8.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "445e881f4f6d382d5f27c034e25eb92edd7c784ceab92a0937db7f2e9471b938"+dependencies = [+ "tinyvec_macros",+]++[[package]]+name = "tinyvec_macros"+version = "0.1.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20"++[[package]]+name = "tokio"+version = "1.39.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "daa4fb1bc778bd6f04cbfc4bb2d06a7396a8f299dc33ea1900cedaa316f467b1"+dependencies = [+ "backtrace",+ "bytes",+ "libc",+ "mio",+ "parking_lot",+ "pin-project-lite",+ "signal-hook-registry",+ "socket2",+ "tokio-macros",+ "windows-sys 0.52.0",+]++[[package]]+name = "tokio-macros"+version = "2.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.72",+]++[[package]]+name = "tokio-native-tls"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2"+dependencies = [+ "native-tls",+ "tokio",+]++[[package]]+name = "tokio-util"+version = "0.7.11"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1"+dependencies = [+ "bytes",+ "futures-core",+ "futures-sink",+ "pin-project-lite",+ "tokio",+]++[[package]]+name = "toml"+version = "0.8.19"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a1ed1f98e3fdc28d6d910e6737ae6ab1a93bf1985935a1193e68f93eeb68d24e"+dependencies = [+ "serde",+ "serde_spanned",+ "toml_datetime",+ "toml_edit",+]++[[package]]+name = "toml_datetime"+version = "0.6.8"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0dd7358ecb8fc2f8d014bf86f6f638ce72ba252a2c3a2572f2a795f1d23efb41"+dependencies = [+ "serde",+]++[[package]]+name = "toml_edit"+version = "0.22.20"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "583c44c02ad26b0c3f3066fe629275e50627026c51ac2e595cca4c230ce1ce1d"+dependencies = [+ "indexmap 2.3.0",+ "serde",+ "serde_spanned",+ "toml_datetime",+ "winnow",+]++[[package]]+name = "tower-service"+version = "0.3.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52"++[[package]]+name = "tracing"+version = "0.1.40"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef"+dependencies = [+ "pin-project-lite",+ "tracing-core",+]++[[package]]+name = "tracing-core"+version = "0.1.32"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c06d3da6113f116aaee68e4d601191614c9053067f9ab7f6edbcb161237daa54"+dependencies = [+ "once_cell",+]++[[package]]+name = "try-lock"+version = "0.2.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"++[[package]]+name = "unicode-bidi"+version = "0.3.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "08f95100a766bf4f8f28f90d77e0a5461bbdb219042e7679bebe79004fed8d75"++[[package]]+name = "unicode-ident"+version = "1.0.12"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"++[[package]]+name = "unicode-normalization"+version = "0.1.23"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a56d1686db2308d901306f92a263857ef59ea39678a5458e7cb17f01415101f5"+dependencies = [+ "tinyvec",+]++[[package]]+name = "unindent"+version = "0.2.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c7de7d73e1754487cb58364ee906a499937a0dfabd86bcb980fa99ec8c8fa2ce"++[[package]]+name = "url"+version = "2.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "22784dbdf76fdde8af1aeda5622b546b422b6fc585325248a2bf9f5e41e94d6c"+dependencies = [+ "form_urlencoded",+ "idna",+ "percent-encoding",+]++[[package]]+name = "utf-8"+version = "0.7.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9"++[[package]]+name = "vcpkg"+version = "0.2.15"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426"++[[package]]+name = "want"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"+dependencies = [+ "try-lock",+]++[[package]]+name = "wasi"+version = "0.11.0+wasi-snapshot-preview1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423"++[[package]]+name = "wasm-bindgen"+version = "0.2.92"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "4be2531df63900aeb2bca0daaaddec08491ee64ceecbee5076636a3b026795a8"+dependencies = [+ "cfg-if",+ "wasm-bindgen-macro",+]++[[package]]+name = "wasm-bindgen-backend"+version = "0.2.92"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "614d787b966d3989fa7bb98a654e369c762374fd3213d212cfc0251257e747da"+dependencies = [+ "bumpalo",+ "log",+ "once_cell",+ "proc-macro2",+ "quote",+ "syn 2.0.72",+ "wasm-bindgen-shared",+]++[[package]]+name = "wasm-bindgen-futures"+version = "0.4.42"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "76bc14366121efc8dbb487ab05bcc9d346b3b5ec0eaa76e46594cabbe51762c0"+dependencies = [+ "cfg-if",+ "js-sys",+ "wasm-bindgen",+ "web-sys",+]++[[package]]+name = "wasm-bindgen-macro"+version = "0.2.92"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a1f8823de937b71b9460c0c34e25f3da88250760bec0ebac694b49997550d726"+dependencies = [+ "quote",+ "wasm-bindgen-macro-support",+]++[[package]]+name = "wasm-bindgen-macro-support"+version = "0.2.92"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "e94f17b526d0a461a191c78ea52bbce64071ed5c04c9ffe424dcb38f74171bb7"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.72",+ "wasm-bindgen-backend",+ "wasm-bindgen-shared",+]++[[package]]+name = "wasm-bindgen-shared"+version = "0.2.92"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "af190c94f2773fdb3729c55b007a722abb5384da03bc0986df4c289bf5567e96"++[[package]]+name = "wasm-streams"+version = "0.4.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "b65dc4c90b63b118468cf747d8bf3566c1913ef60be765b5730ead9e0a3ba129"+dependencies = [+ "futures-util",+ "js-sys",+ "wasm-bindgen",+ "wasm-bindgen-futures",+ "web-sys",+]++[[package]]+name = "web-sys"+version = "0.3.69"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "77afa9a11836342370f4817622a2f0f418b134426d91a82dfb48f532d2ec13ef"+dependencies = [+ "js-sys",+ "wasm-bindgen",+]++[[package]]+name = "windows-sys"+version = "0.48.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9"+dependencies = [+ "windows-targets 0.48.5",+]++[[package]]+name = "windows-sys"+version = "0.52.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"+dependencies = [+ "windows-targets 0.52.6",+]++[[package]]+name = "windows-sys"+version = "0.59.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"+dependencies = [+ "windows-targets 0.52.6",+]++[[package]]+name = "windows-targets"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c"+dependencies = [+ "windows_aarch64_gnullvm 0.48.5",+ "windows_aarch64_msvc 0.48.5",+ "windows_i686_gnu 0.48.5",+ "windows_i686_msvc 0.48.5",+ "windows_x86_64_gnu 0.48.5",+ "windows_x86_64_gnullvm 0.48.5",+ "windows_x86_64_msvc 0.48.5",+]++[[package]]+name = "windows-targets"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"+dependencies = [+ "windows_aarch64_gnullvm 0.52.6",+ "windows_aarch64_msvc 0.52.6",+ "windows_i686_gnu 0.52.6",+ "windows_i686_gnullvm",+ "windows_i686_msvc 0.52.6",+ "windows_x86_64_gnu 0.52.6",+ "windows_x86_64_gnullvm 0.52.6",+ "windows_x86_64_msvc 0.52.6",+]++[[package]]+name = "windows_aarch64_gnullvm"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8"++[[package]]+name = "windows_aarch64_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"++[[package]]+name = "windows_aarch64_msvc"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc"++[[package]]+name = "windows_aarch64_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"++[[package]]+name = "windows_i686_gnu"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e"++[[package]]+name = "windows_i686_gnu"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"++[[package]]+name = "windows_i686_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"++[[package]]+name = "windows_i686_msvc"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406"++[[package]]+name = "windows_i686_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"++[[package]]+name = "windows_x86_64_gnu"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e"++[[package]]+name = "windows_x86_64_gnu"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"++[[package]]+name = "windows_x86_64_gnullvm"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc"++[[package]]+name = "windows_x86_64_gnullvm"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"++[[package]]+name = "windows_x86_64_msvc"+version = "0.48.5"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538"++[[package]]+name = "windows_x86_64_msvc"+version = "0.52.6"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"++[[package]]+name = "winnow"+version = "0.6.18"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "68a9bda4691f099d435ad181000724da8e5899daa10713c2d432552b9ccd3a6f"+dependencies = [+ "memchr",+]++[[package]]+name = "winreg"+version = "0.50.0"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "524e57b2c537c0f9b1e69f1965311ec12182b4122e45035b1508cd24d2adadb1"+dependencies = [+ "cfg-if",+ "windows-sys 0.48.0",+]++[[package]]+name = "zerocopy"+version = "0.7.35"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "1b9b4fd18abc82b8136838da5d50bae7bdea537c574d8dc1a34ed098d6c166f0"+dependencies = [+ "byteorder",+ "zerocopy-derive",+]++[[package]]+name = "zerocopy-derive"+version = "0.7.35"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e"+dependencies = [+ "proc-macro2",+ "quote",+ "syn 2.0.72",+]diff --git a/crates/leechy-py/Cargo.toml b/crates/leechy-py/Cargo.tomlnew file mode 100644index 00000000..f00af3d5--- /dev/null+++ b/crates/leechy-py/Cargo.toml@@ -0,0 +1,17 @@+[package]+edition = "2021"+license = "AGPL-3.0"+name = "leechy"+version = "0.1.0+py"++# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html+[lib]+crate-type = ["cdylib"]+name = "leechy"++[dependencies]+leechy = {path = "../leechy"}+pyo3 = {version = "0.22.0", features = ["extension-module", "experimental-async"]}+tokio = {version = "1.23.1", features = ["full"]}++[workspace]diff --git a/crates/leechy-py/pyproject.toml b/crates/leechy-py/pyproject.tomlnew file mode 100644index 00000000..14fde22c--- /dev/null+++ b/crates/leechy-py/pyproject.toml@@ -0,0 +1,15 @@+[build-system]+build-backend = "maturin"+requires = ["maturin>=1.7,<2.0"]++[project]+classifiers = [+ "Programming Language :: Rust",+ "Programming Language :: Python :: Implementation :: CPython",+ "Programming Language :: Python :: Implementation :: PyPy",+]+dynamic = ["version"]+name = "leechy"+requires-python = ">=3.8"+[tool.maturin]+features = ["pyo3/extension-module"]diff --git a/crates/leechy-py/src/lib.rs b/crates/leechy-py/src/lib.rsnew file mode 100644index 00000000..1cf8b66a--- /dev/null+++ b/crates/leechy-py/src/lib.rs@@ -0,0 +1,61 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use pyo3::prelude::*;++static TOKIO_RUNTIME: std::sync::LazyLock<tokio::runtime::Runtime> =+ std::sync::LazyLock::new(|| {+ tokio::runtime::Builder::new_current_thread()+ .enable_all()+ .build()+ .unwrap()+ });++#[pymodule]+mod leechy {+ use super::*;+ use ::leechy as lchy;++ #[pyclass]+ struct Engine {+ inner: lchy::Engine,+ }++ #[pymethods]+ impl Engine {+ #[new]+ #[pyo3(signature = (name = "startpage"))]+ fn new(name: &str) -> PyResult<Self> {+ match lchy::Engine::by_name(name) {+ Some(engine) => Ok(Self { inner: engine }),+ None => Err(pyo3::exceptions::PyValueError::new_err(format!(+ "Unknown engine: {}",+ name+ ))),+ }+ }++ fn search(&self, query: String) -> PyResult<Vec<String>> {+ TOKIO_RUNTIME.block_on(async {+ self.inner+ .search(query.as_str())+ .await+ .map_err(|e| pyo3::exceptions::PyException::new_err(format!("{}", e)))+ .map(|res| res.into_iter().map(|r| r.to_string()).collect())+ })+ }+ }+}diff --git a/crates/leechy/Cargo.toml b/crates/leechy/Cargo.tomlnew file mode 100644index 00000000..8662e20e--- /dev/null+++ b/crates/leechy/Cargo.toml@@ -0,0 +1,15 @@+[package]+edition = "2021"+license = "AGPL-3.0"+name = "leechy"+version = "0.1.0"++# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html++[dependencies]+anyhow.workspace = true+kuchiki = {path = "../kuchiki"}+rand.workspace = true+reqwest.workspace = true+serde.workspace = true+toml.workspace = truediff --git a/crates/leechy/src/conf.toml b/crates/leechy/src/conf.tomlnew file mode 100644index 00000000..1c492b01--- /dev/null+++ b/crates/leechy/src/conf.toml@@ -0,0 +1,4 @@+[[engines]]+name = "startpage"+search = "https://www.startpage.com/search?q={query}&qsr=en_US"+xpath = "./div[contains(@class, 'result')]/a"diff --git a/crates/leechy/src/engine.rs b/crates/leechy/src/engine.rsnew file mode 100644index 00000000..7b4b939a--- /dev/null+++ b/crates/leechy/src/engine.rs@@ -0,0 +1,91 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use kuchiki::traits::*;+use std::sync::LazyLock;++static PARSED_CONFIG: LazyLock<Config> =+ LazyLock::new(|| toml::from_str(include_str!("conf.toml")).unwrap());++#[derive(serde::Deserialize, Clone)]+pub struct Config {+ engines: Vec<EngineConf>,+}++impl Config {+ pub fn new() -> Self {+ PARSED_CONFIG.clone()+ }+}++#[derive(serde::Deserialize, Clone)]+pub struct EngineConf {+ name: String,+ search: String,+ xpath: String,+}++pub struct Engine {+ conf: EngineConf,+}++impl Engine {+ pub fn new(conf: EngineConf) -> Self {+ Engine { conf }+ }++ pub fn by_name(name: &str) -> Option<Self> {+ let conf = Config::new().engines.into_iter().find(|e| e.name == name)?;+ Some(Engine::new(conf))+ }++ pub async fn search(&self, query: &str) -> Result<Vec<Url>, reqwest::Error> {+ let encoded_query = query.replace(' ', "+");+ let url = self.conf.search.replace("{query}", &encoded_query);+ let useragent = crate::useragent::UserAgent::random_weighted();++ let client = reqwest::Client::builder()+ .user_agent(useragent.as_str())+ .build()?;++ let body = client.get(&url).send().await?.text().await?;++ Ok(kuchiki::parse_html()+ .one(body)+ .select_xpath(&self.conf.xpath)+ .map(|url| {+ let attr = url.attributes.borrow();+ let url = attr.get("href").unwrap();+ Url(url.to_string())+ })+ .collect())+ }+}++#[derive(Debug)]+pub struct Url(String);++impl std::fmt::Display for Url {+ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {+ write!(f, "{}", self.0)+ }+}++impl AsRef<str> for Url {+ fn as_ref(&self) -> &str {+ &self.0+ }+}diff --git a/crates/leechy/src/lib.rs b/crates/leechy/src/lib.rsnew file mode 100644index 00000000..0ab1eec4--- /dev/null+++ b/crates/leechy/src/lib.rs@@ -0,0 +1,20 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++mod engine;+mod useragent;++pub use engine::Engine;diff --git a/crates/leechy/src/useragent.rs b/crates/leechy/src/useragent.rsnew file mode 100644index 00000000..c43a9aa1--- /dev/null+++ b/crates/leechy/src/useragent.rs@@ -0,0 +1,58 @@+// Stract is an open source web search engine.+// Copyright (C) 2024 Stract ApS+//+// This program is free software: you can redistribute it and/or modify+// it under the terms of the GNU Affero General Public License as+// published by the Free Software Foundation, either version 3 of the+// License, or (at your option) any later version.+//+// This program is distributed in the hope that it will be useful,+// but WITHOUT ANY WARRANTY; without even the implied warranty of+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+// GNU Affero General Public License for more details.+//+// You should have received a copy of the GNU Affero General Public License+// along with this program. If not, see <https://www.gnu.org/licenses/>++use rand::prelude::*;+use std::sync::LazyLock;++static USER_AGENTS: LazyLock<Vec<(usize, String)>> = LazyLock::new(|| {+ include_str!("useragents.txt")+ .lines()+ .map(|line| line.trim().to_string())+ .enumerate()+ .collect()+});++pub struct UserAgent(String);++impl UserAgent {+ pub fn random_weighted() -> Self {+ let mut rng = thread_rng();++ UserAgent(+ USER_AGENTS+ .choose_weighted(&mut rng, |(rank, _)| 1 / (rank + 1))+ .unwrap()+ .1+ .clone(),+ )+ }++ pub fn as_str(&self) -> &str {+ &self.0+ }+}++impl std::fmt::Display for UserAgent {+ fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {+ write!(f, "{}", self.0)+ }+}++impl AsRef<str> for UserAgent {+ fn as_ref(&self) -> &str {+ &self.0+ }+}diff --git a/crates/leechy/src/useragents.txt b/crates/leechy/src/useragents.txtnew file mode 100644index 00000000..104e96de--- /dev/null+++ b/crates/leechy/src/useragents.txt@@ -0,0 +1,14 @@+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.1+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.3+Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.3+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.3+Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.3+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/109.0.0.0 Safari/537.3+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 Unique/100.7.6266.6+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36 OPR/112.0.0.+Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:109.0) Gecko/20100101 Firefox/117.+Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.3+Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:109.0) Gecko/20100101 Firefox/115.+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.192 Safari/537.3+Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36 Edg/127.0.0.3\ No newline at end of filediff --git a/ltr/auto_annotate.py b/ltr/auto_annotate.pyindex 718e42ea..2907cf27 100644--- a/ltr/auto_annotate.py+++ b/ltr/auto_annotate.py@@ -7,41 +7,18 @@ import jsonimport timeimport requestsimport random--# API = "https://stract.com/beta/api/search"-API = "http://localhost:3000/beta/api/search"-NUM_RESULTS_PER_QUERY = 20+from db import Db+import stractELO_K = 32ELO_SCALE = 400ELO_ROUNDS_MULT = 5NUM_LABELS = 4-# PROMPT = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>-#-# You are a helpful, smart, kind, and efficient AI assistant. You always fulfill the user's requests to the best of your ability.<|eot_id|><|start_header_id|>user<|end_header_id|>-#-# Think about this step-by-step. You are a search engine evaluator and your task is to evaluate search results based on how well the result matches the query-# You will be shown two results for each query and most choose which result is best for the users query. A good result most answer the users query and come from an authoritative source.-# To choose the best result, write "Best: RESULT_A" or "Best: RESULT_B". Before choosing the best result, you should first evaluate the relevance of each result to the query.-#-# Query: "{}"-#-# RESULT_A:-# Url: "{}"-# Title: "{}"-# Snippet: "{}"-#-# RESULT_B:-# Url: "{}"-# Title: "{}"-# Snippet: "{}"-#-# Evaluation:<|eot_id|><|start_header_id|>assistant<|end_header_id|>-#-# """--PROMPT = """<start_of_turn>user+PROMPT = """<|begin_of_text|><|start_header_id|>system<|end_header_id|>++You are a helpful, smart, kind, and efficient AI assistant. You always fulfill the user's requests to the best of your ability.<|eot_id|><|start_header_id|>user<|end_header_id|>+Think about this step-by-step. You are a search engine evaluator and your task is to evaluate search results based on how well the result matches the queryYou will be shown two results for each query and most choose which result is best for the users query. A good result most answer the users query and come from an authoritative source.To choose the best result, write "Best: RESULT_A" or "Best: RESULT_B". Before choosing the best result, you should first evaluate the relevance of each result to the query.@@ -56,14 +33,16 @@ Snippet: "{}"RESULT_B:Url: "{}"Title: "{}"-Snippet: "{}"<end_of_turn>-<start_of_turn>model+Snippet: "{}"++Evaluation:<|eot_id|><|start_header_id|>assistant<|end_header_id|>+"""+llm = Llama(n_gpu_layers=-1,n_ctx=8000,- # model_path="data/Meta-Llama-3.1-8B-Instruct-Q8_0.gguf",- model_path="data/gemma-2-2b-it-abliterated-Q6_K_L.gguf",+ model_path="data/Meta-Llama-3.1-8B-Instruct-Q8_0.gguf",repeat_penalty=False,no_penalize_nl=True,verbose=False,@@ -77,39 +56,8 @@ with open("data/queries_us.csv") as f:np.random.shuffle(all_queries)-def setup_db():- db = sqlite3.connect("data/auto-ranking-annotation.sqlite")-- cur = db.cursor()-- cur.execute(- """- CREATE TABLE IF NOT EXISTS queries (- qid UUID PRIMARY KEY DEFAULT (HEX(RANDOMBLOB(16))),- query TEXT NOT NULL UNIQUE- );"""- )-- cur.execute(- """- CREATE TABLE IF NOT EXISTS search_results (- qid UUID NOT NULL,- url TEXT NOT NULL,- orig_rank INTEGER NOT NULL,- webpage_json TEXT NOT NULL,- annotation INTEGER DEFAULT NULL,- PRIMARY KEY (qid, url)- );"""- )+db = Db("data/auto-ranking-annotation.sqlite")- db.commit()-- return db---db = setup_db()--cur = db.cursor()for query in all_queries:if len(query) < 3:continue@@ -125,82 +73,15 @@ for query in all_queries:if len(query) > 100:continue- cur.execute(- """- INSERT OR IGNORE INTO queries (query)- VALUES (?)- """,- (query,),- )+ db.add_query(query)-db.commit()-unannotated_queries = cur.execute(- """- SELECT qid, query- FROM queries- WHERE NOT EXISTS (- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL- )- ORDER BY qid- """,-).fetchall()--unannotated_queries = {qid: query for qid, query in unannotated_queries}---def simplify_snippet(snippet):- if "text" not in snippet:- return ""-- return "".join([f["text"] for f in snippet["text"]["fragments"]])---def get_search_results(query):- payload = {- "query": query,- "numResults": NUM_RESULTS_PER_QUERY,- "returnRankingSignals": True,- 'signalCoefficients': {- 'host_centrality_rank': 0.022,- 'page_centrality_rank': 0.022,- }- }-- return [- {- "url": w["url"],- "title": w["title"],- "rankingSignals": {s: v["value"] for (s, v) in w["rankingSignals"].items()},- "snippet": simplify_snippet(w["snippet"]),- }- for w in requests.post(API, json=payload).json()["webpages"][- :NUM_RESULTS_PER_QUERY- ]- ]---def add_results(query):- has_results = (- cur.execute("SELECT 1 FROM search_results WHERE qid = ?", (qid,)).fetchone()- is not None- )- if has_results:- return+unannotated_queries = db.get_unannotated_queries()- results = get_search_results(query)+def add_results(qid, query):+ results = stract.search(query)time.sleep(1)-- for i, result in enumerate(results):- cur.execute(- """- INSERT OR IGNORE INTO search_results (qid, url, orig_rank, webpage_json)- VALUES (?, ?, ?, ?)- """,- (qid, result["url"], i, json.dumps(result)),- )-- db.commit()+ db.insert_results(qid, results)def get_prompt(query, url_a, title_a, snippet_a, url_b, title_b, snippet_b):@@ -233,16 +114,8 @@ def elo_update(winner, loser, elo):for qid, query in tqdm(unannotated_queries.items()):- add_results(query)- unnanotated_results = cur.execute(- """- SELECT url, orig_rank, webpage_json- FROM search_results- WHERE qid = ? AND annotation IS NULL- ORDER BY orig_rank- """,- (qid,),- ).fetchall()+ add_results(qid, query)+ unnanotated_results = db.get_unannotated_results(qid)elo = {url: ELO_SCALE // 2 for url, _, _ in unnanotated_results}@@ -274,7 +147,6 @@ for qid, query in tqdm(unannotated_queries.items()):elo = [{"url": url} for url, _ in elo]for i in range(len(elo)):- # elo[i]['label'] = (NUM_LABELS - i // (len(elo) // NUM_LABELS))elo[i]['label'] = NUM_LABELS - int(np.log2(i + 1))print(query)@@ -282,12 +154,4 @@ for qid, query in tqdm(unannotated_queries.items()):for website in elo:url = website['url']relevancy = website['label']- cur.execute(- """- UPDATE search_results- SET annotation = ?- WHERE qid = ? AND url = ?- """,- (relevancy, qid, url),- )- db.commit()+ db.annotate(qid, url, relevancy)diff --git a/ltr/db.py b/ltr/db.pynew file mode 100644index 00000000..0fad1688--- /dev/null+++ b/ltr/db.py@@ -0,0 +1,98 @@+import sqlite3+import json++class Db:+ def __init__(self, db_path):+ self.db_path = db_path+ self.conn = sqlite3.connect(db_path)+ self.cursor = self.conn.cursor()+ self.setup_db()++ def __del__(self):+ self.cursor.close()+ self.conn.close()++ def setup_db(self):+ self.cursor.execute(+ """+ CREATE TABLE IF NOT EXISTS queries (+ qid UUID PRIMARY KEY DEFAULT (HEX(RANDOMBLOB(16))),+ query TEXT NOT NULL UNIQUE+ );"""+ )++ self.cursor.execute(+ """+ CREATE TABLE IF NOT EXISTS search_results (+ qid UUID NOT NULL,+ url TEXT NOT NULL,+ orig_rank INTEGER NOT NULL,+ webpage_json TEXT NOT NULL,+ annotation INTEGER DEFAULT NULL,+ PRIMARY KEY (qid, url)+ );"""+ )++ self.conn.commit()++ def add_query(self, query: str):+ self.cursor.execute(+ """+ INSERT OR IGNORE INTO queries (query)+ VALUES (?)+ """,+ (query,),+ )+ self.conn.commit()++ def get_unannotated_queries(self):+ unannotated_queries = self.cursor.execute(+ """+ SELECT qid, query+ FROM queries+ WHERE NOT EXISTS (+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL+ )+ ORDER BY qid+ """,+ ).fetchall()++ return {qid: query for qid, query in unannotated_queries}++ def insert_result(self, qid, rank, result):+ self.cursor.execute(+ """+ INSERT OR IGNORE INTO search_results (qid, url, orig_rank, webpage_json)+ VALUES (?, ?, ?, ?)+ """,+ (qid, result["url"], rank, json.dumps(result)),+ )++ self.conn.commit()++ def insert_results(self, qid, results):+ for i, result in enumerate(results):+ self.insert_result(qid, i, result)+++ def get_unannotated_results(self, qid):+ return self.cursor.execute(+ """+ SELECT url, orig_rank, webpage_json+ FROM search_results+ WHERE qid = ? AND annotation IS NULL+ ORDER BY orig_rank+ """,+ (qid,),+ ).fetchall()++ def annotate(self, qid, url, annotation):+ self.cursor.execute(+ """+ UPDATE search_results+ SET annotation = ?+ WHERE qid = ? AND url = ?+ """,+ (annotation, qid, url),+ )+ self.conn.commit()diff --git a/ltr/lambdamart.py b/ltr/lambdamart.pyindex ed3026eb..26a27a30 100644--- a/ltr/lambdamart.py+++ b/ltr/lambdamart.py@@ -29,7 +29,7 @@ res = cur.execute(SELECT qid, queryFROM queriesWHERE EXISTS (- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation > 0)""")diff --git a/ltr/leechy_annotate.py b/ltr/leechy_annotate.pynew file mode 100644index 00000000..f1443efd--- /dev/null+++ b/ltr/leechy_annotate.py@@ -0,0 +1,71 @@+import leechy+from db import Db+import stract+import numpy as np+from tqdm import tqdm+import time++NUM_LABELS = 4++with open("data/queries_us.csv") as f:+ all_queries = [line.strip() for line in f.readlines()]++# shuffle queries+np.random.shuffle(all_queries)++db = Db("data/auto-ranking-annotation.sqlite")++for query in all_queries:+ if len(query) < 3:+ continue++ # check if query has large percentage of non-alphanumeric characters+ if sum([c.isalnum() for c in query]) / len(query) < 0.5:+ continue++ if len(query) > 100:+ continue++ db.add_query(query)+++unannotated_queries = db.get_unannotated_queries()++eng = leechy.Engine()+for qid, query in tqdm(unannotated_queries.items()):+ tqdm.write(query)+ leechy_results = eng.search(query)++ for i, result in enumerate(leechy_results):+ label = NUM_LABELS - int(np.log2(i + 1))++ try:+ data = next(+ res+ for res in stract.search(f"{query} site:{result}")+ if res["url"] == result+ )+ except StopIteration:+ continue++ if not data:+ continue++ tqdm.write(f"{result}: {label}")+ db.insert_result(qid, i, data)+ db.annotate(qid, result, label)++ for page in range(2, 2 + 1):+ bad_results = stract.search(query, page)++ for i, result in enumerate(bad_results):+ if result["url"] in leechy_results:+ continue++ label = 0+ tqdm.write(f'{result["url"]}: {label}')+ db.insert_result(qid, (i + 1) * page, result)+ db.annotate(qid, result["url"], label)++ time.sleep(np.random.normal(60, 20))+ tqdm.write("")diff --git a/ltr/linear_model.py b/ltr/linear_model.pyindex ec2e8346..4a701db9 100644--- a/ltr/linear_model.py+++ b/ltr/linear_model.py@@ -20,7 +20,7 @@ res = cur.execute(SELECT qid, queryFROM queriesWHERE EXISTS (- SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation IS NOT NULL+ SELECT 1 FROM search_results WHERE search_results.qid = queries.qid AND search_results.annotation > 0)""")@@ -108,14 +108,18 @@ y_test = jnp.array(y_test)# jax modelw = jnp.zeros(X_train.shape[1])+def model(w, X):return jnp.dot(X, w)+def loss(w, batch):features, labels, mask = batch- scores = model(w, features)+ # scores = model(w, features)+ # return rax.approx_t12n(rax.mrr_metric)(scores, labels)++ return ((w * features - jnp.ones_like(w)) ** 2).sum()- return rax.approx_t12n(rax.mrr_metric)(scores, labels)grad_fn = jax.jit(jax.grad(loss))@@ -123,12 +127,15 @@ grad_fn = jax.jit(jax.grad(loss))for i in range(128):batch_size = 20for j in range(0, len(X_train), batch_size):- batch = (X_train[j : j + batch_size], y_train[j : j + batch_size], jnp.ones(batch_size))+ batch = (+ X_train[j : j + batch_size],+ y_train[j : j + batch_size],+ jnp.ones(batch_size),+ )w = w - grad_fn(w, batch) * 3e-4w = w.clip(0, 10)-# model = LinearRegression(fit_intercept=False, positive=True)# model = SVR(kernel="linear")# model = DecisionTreeRegressor()@@ -136,7 +143,9 @@ for i in range(128):print("TRAIN")for k in [1, 2, 5, 10]:- print(f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_train], [model(w, X_train)], k=k))+ print(+ f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_train], [model(w, X_train)], k=k)+ )print()print("TEST")@@ -144,7 +153,6 @@ for k in [1, 2, 5, 10]:print(f"NDCG@{k}: ", sklearn.metrics.ndcg_score([y_test], [model(w, X_test)], k=k))-weights = {id2feature[i]: float(v) for (i, v) in enumerate(w)}print()print("Weights:")diff --git a/ltr/stract.py b/ltr/stract.pynew file mode 100644index 00000000..25026f27--- /dev/null+++ b/ltr/stract.py@@ -0,0 +1,32 @@+import requests++# API = "https://stract.com/beta/api/search"+API = "http://localhost:3000/beta/api/search"+NUM_RESULTS_PER_QUERY = 20++def simplify_snippet(snippet):+ if "text" not in snippet:+ return ""++ return "".join([f["text"] for f in snippet["text"]["fragments"]])+++def search(query, page=0):+ payload = {+ "query": query,+ "numResults": NUM_RESULTS_PER_QUERY,+ 'page': page,+ "returnRankingSignals": True,+ }++ return [+ {+ "url": w["url"],+ "title": w["title"],+ "rankingSignals": {s: v["value"] for (s, v) in w["rankingSignals"].items()},+ "snippet": simplify_snippet(w["snippet"]),+ }+ for w in requests.post(API, json=payload).json()["webpages"][+ :NUM_RESULTS_PER_QUERY+ ]+ ]\ No newline at end of file