Skip to content

web search done right

less locking in crawldb

Mikkel Denkercb9a04508bd8
diff --git a/configs/crawler/crawler.toml b/configs/crawler/crawler.toml
index c9874527..d1141329 100644
--- a/configs/crawler/crawler.toml
+++ b/configs/crawler/crawler.toml
@@ -5,7 +5,7 @@ coordinator_host = "0.0.0.0:8080"
[user_agent]
token = "StractBot"
-full = "Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)"
+full = "Mozilla/5.0 (compatible; StractBot/0.1; +https://trystract.com/webmasters)"
[s3]
bucket = "crawl"
diff --git a/core/benches/crawl-coordinator.rs b/core/benches/crawl-coordinator.rs
index ae2960ff..306bd6e2 100644
--- a/core/benches/crawl-coordinator.rs
+++ b/core/benches/crawl-coordinator.rs
@@ -1,1780 +1,71 @@
// rust-analyzer: disable
use criterion::{criterion_group, criterion_main, Criterion};
-use rand::seq::SliceRandom;
-use stract::crawler::{CrawlCoordinator, JobResponse, UrlResponse};
+use rand::Rng;
+use stract::crawler::{CrawlCoordinator, Domain, JobResponse, UrlResponse};
use url::Url;
-pub fn criterion_benchmark(c: &mut Criterion) {
- let coordinator = CrawlCoordinator::new("data/crawldb", 100, vec![]).unwrap();
+fn rand_domain() -> Domain {
+ // generate random domain name by taking from a-z
+ // and appending .com
+ let mut rng = rand::thread_rng();
+ let mut domain = String::new();
+ for _ in 0..rng.gen_range(1..10) {
+ domain.push(rng.gen_range(b'a'..=b'z') as char);
+ }
+ domain.push_str(".com");
+ domain.into()
+}
- let letsencrypt_urls = vec![
-Url::parse("https://letsencrypt.org/2019/11/20/how-le-runs-ct-logs.html").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/invalid-certificate-on-mobile-works-on-desktop/24295").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/frequently-asked-questions-faq").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/upcoming-intermediate-changes/13106").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/san-domain-name-mismatch-android-2-windows-xp/4060").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/small-change-to-end-entity-certificates-cps-url-and-oid-will-not-be-included-from-june-15/198206").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/web-hosting-who-support-lets-encrypt/6920").unwrap(),
-Url::parse("https://letsencrypt.org/2022/04/13/receiving-the-levchin-prize.html").unwrap(),
-Url::parse("https://letsencrypt.org/2020/02/27/one-billion-certs.html").unwrap(),
-Url::parse("https://letsencrypt.org/docs/integration-guide").unwrap(),
-Url::parse("https://letsencrypt.org/fr/docs").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-v3.4").unwrap(),
-Url::parse("https://letsencrypt.org/sr/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/id/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/de/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/he/about").unwrap(),
-Url::parse("https://letsencrypt.org/da/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/es/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/da/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/blog/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/documents/LE-SA-v1.0.1-v1.1.1-diff.docx").unwrap(),
-Url::parse("https://letsencrypt.org/ja/#").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/2015/02/19/draft-isrg-cp.html").unwrap(),
-Url::parse("https://letsencrypt.org/2021/10/21/celebrating-encryption-globally.html").unwrap(),
-Url::parse("https://letsencrypt.org/docs/lencr.org").unwrap(),
-Url::parse("https://letsencrypt.org/it/repository").unwrap(),
-Url::parse("https://community.letsencrypt.org/guidelines").unwrap(),
-Url::parse("https://letsencrypt.org/vi/donate").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/sponsors/#").unwrap(),
-Url::parse("https://letsencrypt.org/fi/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/de/docs").unwrap(),
-Url::parse("https://letsencrypt.org/vi/docs").unwrap(),
-Url::parse("https://letsencrypt.org/docs/glossary/#def-root").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/KOKU9408").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/unauthorized-certificate-issued-for-domain/200845").unwrap(),
-Url::parse("https://letsencrypt.org/contact/#").unwrap(),
-Url::parse("https://letsencrypt.org/2017/10/17/acme-support-in-apache-httpd.html").unwrap(),
-Url::parse("https://letsencrypt.org/fr/donate").unwrap(),
-Url::parse("https://letsencrypt.org/docs/ipv6-support").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/frederik.lohse").unwrap(),
-Url::parse("https://letsencrypt.org/ko/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.3").unwrap(),
-Url::parse("https://letsencrypt.org/fi/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/docs/certificates-for-localhost").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/Oosterkamp").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v3.2").unwrap(),
-Url::parse("https://letsencrypt.org/si/sponsors").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/ssl-certificate-stopped-working/200817").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/dns-dnssec-error-signal/200808").unwrap(),
-Url::parse("https://letsencrypt.org/da/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/docs/staging-environment").unwrap(),
-Url::parse("https://letsencrypt.org/docs/acme-protocol-updates").unwrap(),
-Url::parse("https://letsencrypt.org/si/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/repository").unwrap(),
-Url::parse("https://letsencrypt.org/documents/LE-SA-v1.1.1-v1.2-diff.docx").unwrap(),
-Url::parse("https://letsencrypt.org/de/#").unwrap(),
-Url::parse("https://letsencrypt.org/it/contact").unwrap(),
-Url::parse("https://letsencrypt.org/2016/09/20/what-it-costs-to-run-lets-encrypt.html").unwrap(),
-Url::parse("https://letsencrypt.org/it/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/docs/integration-guide/#when-to-renew").unwrap(),
-Url::parse("https://letsencrypt.org/ko/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/fr/#").unwrap(),
-Url::parse("https://letsencrypt.org/ru/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/hu/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/2015/11/12/public-beta-timing.html/#").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/chandresh").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/apache2-server-certbot-unable-to-find-corresponding-http-vhost/200854").unwrap(),
-Url::parse("https://letsencrypt.org/2015/10/08/isoc-sponsors-lets-encrypt.html").unwrap(),
-Url::parse("https://letsencrypt.org/ja/repository").unwrap(),
-Url::parse("https://letsencrypt.org/fi/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/sr/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/vi/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/2021/02/10/200m-certs-24hrs.html").unwrap(),
-Url::parse("https://letsencrypt.org/hu/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/fr/contact").unwrap(),
-Url::parse("https://letsencrypt.org/docs/too-many-registrations-for-this-ip").unwrap(),
-Url::parse("https://letsencrypt.org/he/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/si/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/sr/stats/#").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/schoen").unwrap(),
-Url::parse("https://letsencrypt.org/2023/01/12/eng-culture-at-isrg.html/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v4.2").unwrap(),
-Url::parse("https://letsencrypt.org/ru/contact").unwrap(),
-Url::parse("https://letsencrypt.org/2014/11/18/announcing-lets-encrypt.html").unwrap(),
-Url::parse("https://letsencrypt.org/id/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/2016/03/09/le-client-new-home.html").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/2015/10/29/phishing-and-malware.html").unwrap(),
-Url::parse("https://letsencrypt.org/id/docs").unwrap(),
-Url::parse("https://letsencrypt.org/ko/sponsors").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/biwek86619").unwrap(),
-Url::parse("https://letsencrypt.org/docs/certificate-compatibility").unwrap(),
-Url::parse("https://letsencrypt.org/fi/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/he/docs").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-v3.2").unwrap(),
-Url::parse("https://letsencrypt.org/ko/stats/#").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/new-certficiate-not-point-to-the-file/200732").unwrap(),
-Url::parse("https://letsencrypt.org/de/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/ja/donate").unwrap(),
-Url::parse("https://letsencrypt.org/de/getinvolved").unwrap(),
-Url::parse("https://community.letsencrypt.org/c/help/ajuda-em-portugues").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/2018/04/04/sct-encoding.html").unwrap(),
-Url::parse("https://letsencrypt.org/about/#").unwrap(),
-Url::parse("https://community.letsencrypt.org/latest?no_definitions=true&page=1").unwrap(),
-Url::parse("https://letsencrypt.org/hu/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/id/#").unwrap(),
-Url::parse("https://letsencrypt.org/hu/contact").unwrap(),
-Url::parse("https://letsencrypt.org/fi/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/he/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/docs/#").unwrap(),
-Url::parse("https://letsencrypt.org/ko/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/uk/#").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/is-it-wise-to-use-lets-encrypt-certificates-for-a-production-endpoint/200836").unwrap(),
-Url::parse("https://letsencrypt.org/he/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/es/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/ko/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/Nummer378").unwrap(),
-Url::parse("https://letsencrypt.org/fi/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/fi/repository").unwrap(),
-Url::parse("https://letsencrypt.org/2023/03/23/improving-resliiency-and-reliability-with-ari.html/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/#").unwrap(),
-Url::parse("https://letsencrypt.org/he/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/de/contact").unwrap(),
-Url::parse("https://letsencrypt.org/si/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/2020/11/06/own-two-feet.html").unwrap(),
-Url::parse("https://letsencrypt.org/he/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/ja/docs").unwrap(),
-Url::parse("https://community.letsencrypt.org/categories").unwrap(),
-Url::parse("https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der").unwrap(),
-Url::parse("https://letsencrypt.org/sv/getting-started").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/josh").unwrap(),
-Url::parse("https://letsencrypt.org/sr/#").unwrap(),
-Url::parse("https://letsencrypt.org/vi/privacy").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/0xffcaee").unwrap(),
-Url::parse("https://letsencrypt.org/da/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/repository").unwrap(),
-Url::parse("https://letsencrypt.org/id/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/he/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/2016/08/05/le-root-to-be-trusted-by-mozilla.html").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/jiapei100").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/jrsyo").unwrap(),
-Url::parse("https://letsencrypt.org/ko/contact").unwrap(),
-Url::parse("https://letsencrypt.org/2022/12/05/ed-letter-2022.html").unwrap(),
-Url::parse("https://letsencrypt.org/docs/rate-limits").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/greatlakesclub").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/o-t-1-options-in-access-log/200818").unwrap(),
-Url::parse("https://letsencrypt.org/ru/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/docs").unwrap(),
-Url::parse("https://letsencrypt.org/si/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/docs/ct-logs").unwrap(),
-Url::parse("https://letsencrypt.org/sv/repository").unwrap(),
-Url::parse("https://letsencrypt.org/2016/11/01/launching-our-crowdfunding-campaign.html").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.7").unwrap(),
-Url::parse("https://letsencrypt.org/de/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/es/donate").unwrap(),
-Url::parse("https://letsencrypt.org/da/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/contact").unwrap(),
-Url::parse("https://letsencrypt.org/it/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/2016/04/01/legal-transparency-report.html").unwrap(),
-Url::parse("https://letsencrypt.org/it/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/fr/stats/#growth").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/openssl-client-compatibility-changes-for-let-s-encrypt-certificates/143816").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.5").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/2017/07/06/wildcard-certificates-coming-jan-2018.html").unwrap(),
-Url::parse("https://letsencrypt.org/fr/about").unwrap(),
-Url::parse("https://letsencrypt.org/de/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/uk/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/2022/09/12/remembering-peter-eckersley.html").unwrap(),
-Url::parse("https://letsencrypt.org/ru/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/2016/04/12/leaving-beta-new-sponsors.html").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/some-vn-cctld-sld-arent-recognized-and-causing-ratelimited-error/200596").unwrap(),
-Url::parse("https://letsencrypt.org/ko/donate").unwrap(),
-Url::parse("https://letsencrypt.org/docs/duplicate-certificate-limit").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/2023/01/12/eng-culture-at-isrg.html/#").unwrap(),
-Url::parse("https://letsencrypt.org/2020/12/21/extending-android-compatibility.html").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/Rip").unwrap(),
-Url::parse("https://letsencrypt.org/de/donate").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/rmhrisk").unwrap(),
-Url::parse("https://community.letsencrypt.org/c/help/ayuda-en-espanol/22").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/acme-v2-production-environment-wildcards/55578").unwrap(),
-Url::parse("https://letsencrypt.org/vi/stats/#").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/tls-1-2-and-tls-1-3-need-curve25519-and-curve448-ssl-certificates/200775").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/dns-problem-query-timed-out-looking-up-a/200751").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/Nekit").unwrap(),
-Url::parse("https://letsencrypt.org/privacy/#").unwrap(),
-Url::parse("https://letsencrypt.org/fi/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/vi/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/ru/donate").unwrap(),
-Url::parse("https://letsencrypt.org/da/#").unwrap(),
-Url::parse("https://letsencrypt.org/vi/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v3.1").unwrap(),
-Url::parse("https://letsencrypt.org/2015/08/13/lets-encrypt-community-support.html").unwrap(),
-Url::parse("https://letsencrypt.org/vi/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/fr/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/ru/docs/faq").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/is-there-anything-new-to-generate-lets-encrypt-certificate-with-docker-and-nginx/200837").unwrap(),
-Url::parse("https://letsencrypt.org/uk/docs").unwrap(),
-Url::parse("https://letsencrypt.org/si/about").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/certbot-renew-task-windows-setting/200282").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/keeptake").unwrap(),
-Url::parse("https://letsencrypt.org/vi/about").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/docs").unwrap(),
-Url::parse("https://letsencrypt.org/docs/faq/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.1").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/Bruce5051").unwrap(),
-Url::parse("https://letsencrypt.org/si/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/ru/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/sr/about").unwrap(),
-Url::parse("https://letsencrypt.org/da/getinvolved").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/why-i-cant-acess-this-site/200850").unwrap(),
-Url::parse("https://letsencrypt.org/2015/12/03/facebook-sponsorship.html").unwrap(),
-Url::parse("https://cp.letsencrypt.org").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/docs").unwrap(),
-Url::parse("https://letsencrypt.org/id/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.3").unwrap(),
-Url::parse("https://letsencrypt.org/fr/repository").unwrap(),
-Url::parse("https://letsencrypt.org/sv/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/2015/11/12/public-beta-timing.html/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/donate").unwrap(),
-Url::parse("https://letsencrypt.org/ja/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/sr/docs/faq").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/jacordar").unwrap(),
-Url::parse("https://letsencrypt.org/sr/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/es/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/2017/01/06/le-2016-in-review.html").unwrap(),
-Url::parse("https://letsencrypt.org/2022/02/25/ford-foundation.html").unwrap(),
-Url::parse("https://letsencrypt.org/2023/03/23/improving-resliiency-and-reliability-with-ari.html/#").unwrap(),
-Url::parse("https://letsencrypt.org/si/#").unwrap(),
-Url::parse("https://letsencrypt.org/docs/caa").unwrap(),
-Url::parse("https://letsencrypt.org/es/docs").unwrap(),
-Url::parse("https://letsencrypt.org/2016/10/01/legal-transparency-report.html").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.6").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/#").unwrap(),
-Url::parse("https://letsencrypt.org/fi/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/hu/stats/#").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/mlen").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/about").unwrap(),
-Url::parse("https://letsencrypt.org/sv/about").unwrap(),
-Url::parse("https://letsencrypt.org/docs/dst-root-ca-x3-expiration-september-2021/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/ja/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/sv/#").unwrap(),
-Url::parse("https://letsencrypt.org/ru/docs").unwrap(),
-Url::parse("https://letsencrypt.org/docs/allow-port-80").unwrap(),
-Url::parse("https://letsencrypt.org/2015/06/04/isrg-ca-certs.html").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v4.5").unwrap(),
-Url::parse("https://letsencrypt.org/es/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/2015/12/03/entering-public-beta.html/#").unwrap(),
-Url::parse("https://letsencrypt.org/2016/10/21/introducing-idn-support.html").unwrap(),
-Url::parse("https://letsencrypt.org/uk/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.6").unwrap(),
-Url::parse("https://letsencrypt.org/certificates").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/thunderbird-exim-dovecot-cert-settings/200882").unwrap(),
-Url::parse("https://letsencrypt.org/fi/donate").unwrap(),
-Url::parse("https://letsencrypt.org/ru/sponsors").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/rahvee_tufts").unwrap(),
-Url::parse("https://letsencrypt.org/ko/docs").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-cps-v5.0").unwrap(),
-Url::parse("https://letsencrypt.org/ru/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/contact").unwrap(),
-Url::parse("https://letsencrypt.org/privacy/#main-content").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/lordspace").unwrap(),
-Url::parse("https://letsencrypt.org/docs/glossary").unwrap(),
-Url::parse("https://letsencrypt.org/sr/docs").unwrap(),
-Url::parse("https://letsencrypt.org/sv/docs").unwrap(),
-Url::parse("https://letsencrypt.org/2020/09/17/new-root-and-intermediates.html").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/he/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.5").unwrap(),
-Url::parse("https://letsencrypt.org/uk/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/fr/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/2015/06/16/lets-encrypt-launch-schedule.html").unwrap(),
-Url::parse("https://letsencrypt.org/uk/docs/faq").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/welcome-to-lets-encrypt-community-support/8").unwrap(),
-Url::parse("https://letsencrypt.org/2019/10/09/onboarding-your-customers-with-lets-encrypt-and-acme.html").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/no-renewal-config-found/200878").unwrap(),
-Url::parse("https://letsencrypt.org/it/donate").unwrap(),
-Url::parse("https://letsencrypt.org/he/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/id/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/hu/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v4.1").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/reminder-email-after-adding-or-removing-names/200846").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/dh-2048-to-dh-4096/200885").unwrap(),
-Url::parse("https://letsencrypt.org/hu/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/ko/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/docs/godaddy").unwrap(),
-Url::parse("https://letsencrypt.org/ja/about").unwrap(),
-Url::parse("https://letsencrypt.org/2021/10/28/tls-simply-and-automatically.html").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.0").unwrap(),
-Url::parse("https://letsencrypt.org/2018/12/31/looking-forward-to-2019.html").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/stats/#growth").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/kaponir").unwrap(),
-Url::parse("https://letsencrypt.org/sr/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/sr/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/vi/contact").unwrap(),
-Url::parse("https://letsencrypt.org/2016/10/27/first-grant-ford-foundation.html").unwrap(),
-Url::parse("https://letsencrypt.org/sv/stats/#main-content").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/multiple-ssl-with-a-single-mail-server/200804").unwrap(),
-Url::parse("https://letsencrypt.org/it/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/fi/#").unwrap(),
-Url::parse("https://letsencrypt.org/da/docs").unwrap(),
-Url::parse("https://community.letsencrypt.org/c/issuance-tech-questions/12").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/mastodon-type-unauthorized-invalid-response-from-404/200815").unwrap(),
-Url::parse("https://letsencrypt.org/docs/challenge-types/#dns-01-challenge").unwrap(),
-Url::parse("https://cps.letsencrypt.org").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v3.0").unwrap(),
-Url::parse("https://letsencrypt.org/he/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/fr/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/es/#").unwrap(),
-Url::parse("https://letsencrypt.org/uk/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/2018/08/06/trusted-by-all-major-root-programs.html").unwrap(),
-Url::parse("https://letsencrypt.org/id/about").unwrap(),
-Url::parse("https://letsencrypt.org/id/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/sv/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/2023/01/19/renewing-sponsors.html/#").unwrap(),
-Url::parse("https://letsencrypt.org/2019/05/15/introducing-oak-ct-log.html").unwrap(),
-Url::parse("https://letsencrypt.org/ja/stats/#growth").unwrap(),
-Url::parse("https://community.letsencrypt.org/latest.rss").unwrap(),
-Url::parse("https://letsencrypt.org/2015/04/09/isrg-lf-collaboration.html").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/fr/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/ru/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/he/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/docs/account-id").unwrap(),
-Url::parse("https://letsencrypt.org/da/about").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/da/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/de/about").unwrap(),
-Url::parse("https://letsencrypt.org/docs/challenge-types").unwrap(),
-Url::parse("https://letsencrypt.org/2015/08/07/updated-lets-encrypt-launch-schedule.html").unwrap(),
-Url::parse("https://letsencrypt.org/it/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/uk/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/hu/donate").unwrap(),
-Url::parse("https://letsencrypt.org/sr/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/fr/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/id/contact").unwrap(),
-Url::parse("https://letsencrypt.org/de/privacy").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/help-thread-for-dst-root-ca-x3-expiration-september-2021/149190").unwrap(),
-Url::parse("https://letsencrypt.org/ru/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/sv/donate").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.9").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/repository").unwrap(),
-Url::parse("https://letsencrypt.org/hu/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/hu/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/repository/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/ko/#").unwrap(),
-Url::parse("https://letsencrypt.org/2015/10/19/lets-encrypt-is-trusted.html").unwrap(),
-Url::parse("https://letsencrypt.org/sv/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v4.0").unwrap(),
-Url::parse("https://letsencrypt.org/he/stats/#").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/xtronics").unwrap(),
-Url::parse("https://letsencrypt.org/es/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/id/privacy").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/robouden").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/ko/#main-content").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/Osiris").unwrap(),
-Url::parse("https://letsencrypt.org/docs/#main-content").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/jvanasco").unwrap(),
-Url::parse("https://letsencrypt.org/2015/12/21/ovh-sponsorship.html").unwrap(),
-Url::parse("https://letsencrypt.org/si/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/es/contact").unwrap(),
-Url::parse("https://letsencrypt.org/es/about").unwrap(),
-Url::parse("https://letsencrypt.org/2016/03/08/our-millionth-cert.html").unwrap(),
-Url::parse("https://letsencrypt.org/docs/revoking").unwrap(),
-Url::parse("https://letsencrypt.org/si/repository").unwrap(),
-Url::parse("https://letsencrypt.org/de/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/vi/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/mholt").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-v3.0").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/renewal-not-avaiable/200869").unwrap(),
-Url::parse("https://letsencrypt.org/de/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/2019/04/15/transitioning-to-isrg-root.html").unwrap(),
-Url::parse("https://letsencrypt.org/2021/09/14/speed-at-scale-shopify.html").unwrap(),
-Url::parse("https://community.letsencrypt.org/c/client-dev/14").unwrap(),
-Url::parse("https://letsencrypt.org/getinvolved/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/ja/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/ru/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/id/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/it/#main-content").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/_az").unwrap(),
-Url::parse("https://letsencrypt.org/it/docs").unwrap(),
-Url::parse("https://letsencrypt.org/ko/repository").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/rg305").unwrap(),
-Url::parse("https://letsencrypt.org/id/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/2022/04/28/database-to-app-tls.html").unwrap(),
-Url::parse("https://letsencrypt.org/uk/trademarks").unwrap(),
-Url::parse("https://letsencrypt.org/sr/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/fi/about").unwrap(),
-Url::parse("https://letsencrypt.org/id/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/it/about").unwrap(),
-Url::parse("https://letsencrypt.org/fi/#main-content").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/linkp").unwrap(),
-Url::parse("https://community.letsencrypt.org/tos").unwrap(),
-Url::parse("https://letsencrypt.org/ja/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/upcoming-features").unwrap(),
-Url::parse("https://letsencrypt.org/uk/donate").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/aarongable").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/Mahmoud1").unwrap(),
-Url::parse("https://letsencrypt.org/sr/stats/#percent-pageloads").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/yyny").unwrap(),
-Url::parse("https://letsencrypt.org/ja/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/petercooperjr").unwrap(),
-Url::parse("https://letsencrypt.org/da/donate").unwrap(),
-Url::parse("https://letsencrypt.org/id/repository").unwrap(),
-Url::parse("https://letsencrypt.org/si/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/vi/#").unwrap(),
-Url::parse("https://letsencrypt.org/it/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/es/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/id/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/hu/docs").unwrap(),
-Url::parse("https://letsencrypt.org/ko/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/donate/#").unwrap(),
-Url::parse("https://letsencrypt.org/blog/#").unwrap(),
-Url::parse("https://letsencrypt.org/vi/stats/#growth").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/cocolinofan").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.0").unwrap(),
-Url::parse("https://letsencrypt.org/sv/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/fr/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/de/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/ru/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/si/docs").unwrap(),
-Url::parse("https://letsencrypt.org/getinvolved/#").unwrap(),
-Url::parse("https://letsencrypt.org/hu/#").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/2023/05/24/isrg-10th-anniversary.html/#").unwrap(),
-Url::parse("https://letsencrypt.org/2019/02/12/facebook-expands-support-for-letsencrypt.html").unwrap(),
-Url::parse("https://letsencrypt.org/hu/repository").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/production-chain-changes/150739/4").unwrap(),
-Url::parse("https://letsencrypt.org/docs/dst-root-ca-x3-expiration-september-2021/#").unwrap(),
-Url::parse("https://letsencrypt.org/ko/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/ja/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/2021/12/16/ed-letter-2021.html").unwrap(),
-Url::parse("https://letsencrypt.org/2017/03/23/ovh-platinum-renewal.html").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/smyasoedov").unwrap(),
-Url::parse("https://letsencrypt.org/es/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/2016/06/22/https-progress-june-2016.html").unwrap(),
-Url::parse("https://letsencrypt.org/2021/01/21/next-gen-database-servers.html").unwrap(),
-Url::parse("https://letsencrypt.org/da/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/fr/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/2016/07/26/full-ipv6-support.html").unwrap(),
-Url::parse("https://letsencrypt.org/he/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/2022/05/19/nurturing-ct-log-growth.html").unwrap(),
-Url::parse("https://letsencrypt.org/2015/07/01/legal-transparency-report.html").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.1").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/certbot-invalid-response-http-www-example-org-well-known-acme-challenge/200841").unwrap(),
-Url::parse("https://letsencrypt.org/hu/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/sr/repository").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/production-chain-changes/150739").unwrap(),
-Url::parse("https://letsencrypt.org/docs/faq/#").unwrap(),
-Url::parse("https://letsencrypt.org/es/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/vi/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/security_letsencrypt.org-publickey.asc").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/using-staging-and-certbot-saves-stuff-in-live-directory/200780").unwrap(),
-Url::parse("https://letsencrypt.org/2023/01/19/renewing-sponsors.html/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/contact/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/2015/04/14/ncc-group-audit.html").unwrap(),
-Url::parse("https://letsencrypt.org/it/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/ja/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/2017/12/07/looking-forward-to-2018.html").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/about").unwrap(),
-Url::parse("https://letsencrypt.org/sr/contact").unwrap(),
-Url::parse("https://letsencrypt.org/2015/11/09/why-90-days.html/#").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/uk/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-v3.3").unwrap(),
-Url::parse("https://letsencrypt.org/sr/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/si/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/2017/06/14/acme-v2-api.html").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/about").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-v3.1").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/hello-can-i-use-lets-encrypt-for-postfix/200169").unwrap(),
-Url::parse("https://letsencrypt.org/hu/about").unwrap(),
-Url::parse("https://letsencrypt.org/getting-started/#").unwrap(),
-Url::parse("https://letsencrypt.org/2015/12/03/entering-public-beta.html/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/sv/contact").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v4.4").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.2").unwrap(),
-Url::parse("https://letsencrypt.org/docs/failed-validation-limit").unwrap(),
-Url::parse("https://letsencrypt.org/sr/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/fr/stats/#").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/system").unwrap(),
-Url::parse("https://community.letsencrypt.org/c/issuance-policy/11").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/knX5Fw").unwrap(),
-Url::parse("https://letsencrypt.org/da/repository").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/acmeisuptime-com/200022").unwrap(),
-Url::parse("https://letsencrypt.org/ru/repository").unwrap(),
-Url::parse("https://letsencrypt.org/2015/05/21/draft-le-sa.html").unwrap(),
-Url::parse("https://letsencrypt.org/how-it-works").unwrap(),
-Url::parse("https://letsencrypt.org/vi/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/fi/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.4").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/uk/contact").unwrap(),
-Url::parse("https://letsencrypt.org/fr/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/it/#").unwrap(),
-Url::parse("https://letsencrypt.org/2023/05/24/isrg-10th-anniversary.html/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/vi/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/2015/09/14/our-first-cert.html").unwrap(),
-Url::parse("https://letsencrypt.org/es/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.4").unwrap(),
-Url::parse("https://letsencrypt.org/ko/about").unwrap(),
-Url::parse("https://letsencrypt.org/he/contact").unwrap(),
-Url::parse("https://letsencrypt.org/uk/about").unwrap(),
-Url::parse("https://letsencrypt.org/it/stats/#main-content").unwrap(),
-Url::parse("https://community.letsencrypt.org/c/help/aide-en-francais").unwrap(),
-Url::parse("https://letsencrypt.org/sponsors/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/sv/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v3.3").unwrap(),
-Url::parse("https://letsencrypt.org/he/#").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-cps-v5.1").unwrap(),
-Url::parse("https://letsencrypt.org/fi/stats/#percent-pageloads").unwrap(),
-Url::parse("https://community.letsencrypt.org/posts.rss").unwrap(),
-Url::parse("https://letsencrypt.org/documents/LE-SA-v1.2-v1.3-diff.docx").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/chit164").unwrap(),
-Url::parse("https://letsencrypt.org/da/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/uk/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/si/contact").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/stats/#daily-issuance").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/renew-dry-run-fails-with-unable-to-determine-base-domain-error/200831").unwrap(),
-Url::parse("https://letsencrypt.org/sv/stats/#percent-pageloads").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/webprofusion").unwrap(),
-Url::parse("https://letsencrypt.org/uk/repository").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.2").unwrap(),
-Url::parse("https://letsencrypt.org/uk/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/ja/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/2016/10/24/squarespace-ocsp-impl.html").unwrap(),
-Url::parse("https://letsencrypt.org/de/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/2016/06/23/defending-our-brand.html").unwrap(),
-Url::parse("https://letsencrypt.org/2015/04/23/draft-isrg-cp-cps.html").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/orangepizza").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/problema-de-conexion-connection-problem/200673").unwrap(),
-Url::parse("https://letsencrypt.org/id/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/docs/expiration-emails").unwrap(),
-Url::parse("https://letsencrypt.org/it/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/uk/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/es/repository").unwrap(),
-Url::parse("https://letsencrypt.org/donate/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/de/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/donate").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.7").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/JoseGR02").unwrap(),
-Url::parse("https://letsencrypt.org/2020/12/28/executive-director-letter.html").unwrap(),
-Url::parse("https://letsencrypt.org/ja/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/contact").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/dns-01-challenge-start-to-failure-since-24-at-22h/200816").unwrap(),
-Url::parse("https://letsencrypt.org/ru/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/ko/stats/#percent-pageloads").unwrap(),
-Url::parse("https://community.letsencrypt.org/opensearch.xml").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/fi/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/hu/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/2020/02/19/multi-perspective-validation.html").unwrap(),
-Url::parse("https://letsencrypt.org/docs/client-options").unwrap(),
-Url::parse("https://letsencrypt.org/uk/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/getting-started/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/repository/#").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.8").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/thoughts-from-starting-to-play-with-ari/200276").unwrap(),
-Url::parse("https://letsencrypt.org/ru/#").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/Krischu").unwrap(),
-Url::parse("https://letsencrypt.org/sr/donate").unwrap(),
-Url::parse("https://letsencrypt.org/docs/faq/#technical").unwrap(),
-Url::parse("https://letsencrypt.org/si/stats/#main-content").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/vrubiella").unwrap(),
-Url::parse("https://letsencrypt.org/fi/contact").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/stats/#main-content").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/MikeMcQ").unwrap(),
-Url::parse("https://letsencrypt.org/es/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/he/repository").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/mcpherrinm").unwrap(),
-Url::parse("https://letsencrypt.org/vi/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/id/donate").unwrap(),
-Url::parse("https://letsencrypt.org/da/stats/#daily-issuance").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/phonegal2").unwrap(),
-Url::parse("https://letsencrypt.org/sv/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/si/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/2019/03/11/acme-protocol-ietf-standard.html").unwrap(),
-Url::parse("https://letsencrypt.org/si/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/fi/docs").unwrap(),
-Url::parse("https://letsencrypt.org/2022/12/15/ocspcaching.html/#").unwrap(),
-Url::parse("https://letsencrypt.org/da/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/he/donate").unwrap(),
-Url::parse("https://letsencrypt.org/docs/faq/#general").unwrap(),
-Url::parse("https://letsencrypt.org/fr/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/2015/11/09/why-90-days.html/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/fr/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/donate").unwrap(),
-Url::parse("https://letsencrypt.org/vi/repository").unwrap(),
-Url::parse("https://letsencrypt.org/ja/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/ko/stats/#main-content").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/mason8592").unwrap(),
-Url::parse("https://letsencrypt.org/2022/09/07/new-life-for-crls.html").unwrap(),
-Url::parse("https://letsencrypt.org/2022/12/15/ocspcaching.html/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/ja/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/2017/06/28/hundred-million-certs.html").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/uk/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/da/contact").unwrap(),
-Url::parse("https://letsencrypt.org/hu/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/de/repository").unwrap(),
-Url::parse("https://letsencrypt.org/it/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/es/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/it/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/da/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/es/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/hu/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/si/donate").unwrap(),
-Url::parse("https://letsencrypt.org/about/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/de/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/ru/about").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/letsuser").unwrap(),
-Url::parse("https://community.letsencrypt.org/u/nigihayami").unwrap(),
-Url::parse("https://letsencrypt.org/sv/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/sv/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://community.letsencrypt.org/c/help/13").unwrap(),
-Url::parse("https://letsencrypt.org/sv/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/documents/isrg-cps-v4.3").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/#").unwrap(),
-Url::parse("https://letsencrypt.org/trademarks").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/2021/10/01/cert-chaining-help.html").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/get-error-ref-5401-then-now-ref-5017/200867").unwrap(),
-Url::parse("https://letsencrypt.org/ja/contact").unwrap(),
-Url::parse("https://www.letsencrypt.org").unwrap(),
-Url::parse("https://letsencrypt.org/ja/docs/allow-port-80").unwrap(),
-Url::parse("https://letsencrypt.org/docs/rate-limits#:~:text=limit%20of%205%20per%20week").unwrap(),
-Url::parse("https://letsencrypt.org/docs/faq/#what-ip-addresses-does-let-s-encrypt-use-to-validate-my-web-server").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/revoking-certain-certificates-on-march-4/114864").unwrap(),
-Url::parse("https://letsencrypt.org/become-a-sponsor").unwrap(),
-Url::parse("https://letsencrypt.org/?ref=blog.sean-wright.com").unwrap(),
-Url::parse("https://letsencrypt.org/?ref=spreadprivacy.com").unwrap(),
-Url::parse("https://letsencrypt.org/howitworks/technology").unwrap(),
-Url::parse("https://community.letsencrypt.org/t/beta-program-announcements/1631/6").unwrap(),
-Url::parse("https://acme-v01.api.letsencrypt.org/directory").unwrap(),
-Url::parse("https://acme-v01.api.letsencrypt.org/directory%22").unwrap(),
-Url::parse("https://letsencrypt.org/feed.xml").unwrap(),
-Url::parse("https://letsencrypt.org/stats").unwrap(),
-Url::parse("https://letsencrypt.org").unwrap(),
-Url::parse("https://letsencrypt.org/2015/11/12/public-beta-timing.html").unwrap(),
-Url::parse("https://letsencrypt.org/2015/12/03/entering-public-beta.html").unwrap(),
-Url::parse("https://letsencrypt.org/2015/11/09/why-90-days.html").unwrap(),
-Url::parse("https://letsencrypt.org/2023/03/23/improving-resliiency-and-reliability-with-ari.html").unwrap(),
-Url::parse("https://letsencrypt.org/vi").unwrap(),
-Url::parse("https://letsencrypt.org/blog").unwrap(),
-Url::parse("https://letsencrypt.org/stats/#growth").unwrap(),
-Url::parse("https://letsencrypt.org/si").unwrap(),
-Url::parse("https://letsencrypt.org/docs/faq").unwrap(),
-Url::parse("https://letsencrypt.org/uk/stats").unwrap(),
-Url::parse("https://letsencrypt.org/contact").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw").unwrap(),
-Url::parse("https://letsencrypt.org/he").unwrap(),
-Url::parse("https://letsencrypt.org/stats/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/fi").unwrap(),
-Url::parse("https://letsencrypt.org/stats/#daily-issuance").unwrap(),
-Url::parse("https://letsencrypt.org/vi/stats").unwrap(),
-Url::parse("https://letsencrypt.org/fr/stats").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br").unwrap(),
-Url::parse("https://letsencrypt.org/stats/#").unwrap(),
-Url::parse("https://letsencrypt.org/#main-content").unwrap(),
-Url::parse("https://letsencrypt.org/privacy").unwrap(),
-Url::parse("https://letsencrypt.org/sr").unwrap(),
-Url::parse("https://letsencrypt.org/id").unwrap(),
-Url::parse("https://letsencrypt.org/#").unwrap(),
-Url::parse("https://letsencrypt.org/docs").unwrap(),
-Url::parse("https://letsencrypt.org/sr/stats").unwrap(),
-Url::parse("https://letsencrypt.org/hu").unwrap(),
-Url::parse("https://letsencrypt.org/he/stats").unwrap(),
-Url::parse("https://letsencrypt.org/sponsors").unwrap(),
-Url::parse("https://letsencrypt.org/ja/stats").unwrap(),
-Url::parse("https://letsencrypt.org/getting-started").unwrap(),
-Url::parse("https://letsencrypt.org/de/stats").unwrap(),
-Url::parse("https://letsencrypt.org/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
-Url::parse("https://letsencrypt.org/sv/stats").unwrap(),
-Url::parse("https://letsencrypt.org/hu/stats").unwrap(),
-Url::parse("https://letsencrypt.org/es").unwrap(),
-Url::parse("https://letsencrypt.org/it").unwrap(),
-Url::parse("https://letsencrypt.org/it/stats").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn/stats").unwrap(),
-Url::parse("https://letsencrypt.org/sv").unwrap(),
-Url::parse("https://letsencrypt.org/getinvolved").unwrap(),
-Url::parse("https://letsencrypt.org/ru").unwrap(),
-Url::parse("https://letsencrypt.org/repository").unwrap(),
-Url::parse("https://letsencrypt.org/zh-cn").unwrap(),
-Url::parse("https://letsencrypt.org/fr").unwrap(),
-Url::parse("https://letsencrypt.org/da").unwrap(),
-Url::parse("https://letsencrypt.org/about").unwrap(),
-Url::parse("https://letsencrypt.org/ru/stats").unwrap(),
-Url::parse("https://letsencrypt.org/ko/stats").unwrap(),
-Url::parse("https://letsencrypt.org/stats/#percent-pageloads").unwrap(),
-Url::parse("https://letsencrypt.org/ko").unwrap(),
-Url::parse("https://letsencrypt.org/2022/12/15/ocspcaching.html").unwrap(),
-Url::parse("https://community.letsencrypt.org").unwrap(),
-Url::parse("https://letsencrypt.org/si/stats").unwrap(),
-Url::parse("https://letsencrypt.org/es/stats").unwrap(),
-Url::parse("https://letsencrypt.org/pt-br/stats").unwrap(),
-Url::parse("https://letsencrypt.org/uk").unwrap(),
-Url::parse("https://letsencrypt.org/de").unwrap(),
-Url::parse("https://letsencrypt.org/donate").unwrap(),
-Url::parse("https://letsencrypt.org/fi/stats").unwrap(),
-Url::parse("https://letsencrypt.org/2023/01/19/renewing-sponsors.html").unwrap(),
-Url::parse("https://letsencrypt.org/ja").unwrap(),
-Url::parse("https://letsencrypt.org/da/stats").unwrap(),
-Url::parse("https://letsencrypt.org/2023/01/12/eng-culture-at-isrg.html").unwrap(),
-Url::parse("https://letsencrypt.org/id/stats").unwrap(),
-Url::parse("https://letsencrypt.org/zh-tw/stats").unwrap(),
-Url::parse("https://letsencrypt.org/2023/05/24/isrg-10th-anniversary.html").unwrap(),
- ];
+fn rand_url(domain: &Domain) -> Url {
+ let mut path = String::new();
- let apple_urls = vec![
-Url::parse("https://www.apple.com/newsroom/2023/06/developer-tools-to-create-spatial-experiences-for-apple-vision-pro-now-available").unwrap(),
-Url::parse("https://apps.apple.com/us/app/top10-com/id1545027369").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/safari-to-go/id881697395").unwrap(),
-Url::parse("https://support.apple.com/fr-fr/safari").unwrap(),
-Url::parse("https://apps.apple.com/app/id1145737091").unwrap(),
-Url::parse("https://itunes.apple.com/app/id473512078").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/articles/id364881979").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/geosearch-for-wikipedia/id929984054").unwrap(),
-Url::parse("https://itunes.apple.com/eg/app/wikiflip/id1156078290?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/app/wikiwand-faster-wikipedia-reader/id947444965").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/learn-discovery-mindmap-wikipedia/id859954749").unwrap(),
-Url::parse("https://apps.apple.com/us/app/duovocal-learn-german/id1548499329").unwrap(),
-Url::parse("https://itunes.apple.com/app/viki-wikipedia/id993435362").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/wikimed/id1281693200?mt=8").unwrap(),
-Url::parse("https://apps.apple.com/us/app/miniwiki/id1448143759#?platform=appleWatch").unwrap(),
-Url::parse("https://itunes.apple.com/app/wikicompass/id1363789167").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/inquire-by-tamper/id1045268097").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/kiwix/id997079563").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/wikinodes/id433834594").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/id1046122820").unwrap(),
-Url::parse("https://itunes.apple.com/en/app/wikipanion/id288349436").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/wikiweb-visual-wikipedia-reader/id499345601").unwrap(),
-Url::parse("https://itunes.apple.com/eg/app/wikimaster/id1062887157?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/tech-xplore/id1154953172?mt=8").unwrap(),
-Url::parse("https://developer.apple.com/documentation/xcode/writing-armv6-code-for-ios").unwrap(),
-Url::parse("https://apps.apple.com/us/app/polygit-git-client/id1508942822").unwrap(),
-Url::parse("https://apps.apple.com/ro/app/cinemagia-mobile/id445358827").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/id1082193172?mt=8").unwrap(),
-Url::parse("https://help.apple.com/systemimageutility/mac/10.12/#/sysmfd460053").unwrap(),
-Url::parse("https://support.apple.com/en-gb/HT208050").unwrap(),
-Url::parse("https://developer.apple.com/documentation/swiftdata/relationship").unwrap(),
-Url::parse("https://developer.apple.com/documentation/SwiftData").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/when-to-use-microservices-when-not-to-sam-newman-martin/id1559093928?i=1000514252180").unwrap(),
-Url::parse("http://support.apple.com/kb/HT5012").unwrap(),
-Url::parse("http://www.apple.com/certificateauthority/ca_program.html").unwrap(),
-Url::parse("http://support.apple.com/kb/HT6005").unwrap(),
-Url::parse("http://www.apple.com").unwrap(),
-Url::parse("http://support.apple.com/kb/HT1677?viewlocale=da_DK").unwrap(),
-Url::parse("http://maps.apple.com/?q=Highgate+Cemetery,+Swains+Lane,+London+N6+6PJ").unwrap(),
-Url::parse("http://itunes.apple.com/us/book/isbn9780990582915").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/git-drive/id1215974700?ls=1&mt=8").unwrap(),
-Url::parse("https://apple.com").unwrap(),
-Url::parse("https://www.apple.com/ca/osx/apps/app-store/index.html").unwrap(),
-Url::parse("https://support.apple.com/de-de/guide/safari/sfri11471/mac").unwrap(),
-Url::parse("https://support.apple.com/sv-se/guide/iphone/iph96b214f0/ios").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/wikipedia-mobile/id324715238?mt=8").unwrap(),
-Url::parse("https://testflight.apple.com/join/Z0AU0KXC").unwrap(),
-Url::parse("https://itunes.apple.com/us/institution/library-of-congress/id386017159").unwrap(),
-Url::parse("https://itunes.apple.com/us/artist/library-of-congress/id1280713133?mt=2").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/new-york-post-ipad-edition/id378590820?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/new-york-post-iphone-edition/id410094240?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/au/app/pocket-casts/id414834813").unwrap(),
-Url::parse("https://www.apple.com/apple-watch-ultra").unwrap(),
-Url::parse("http://support.apple.com/kb/PH5042").unwrap(),
-Url::parse("http://www.apple.com/macos").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT208331").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT205030").unwrap(),
-Url::parse("https://itunes.apple.com/us/artist/id2080713").unwrap(),
-Url::parse("https://www.apple.com/osx").unwrap(),
-Url::parse("http://support.apple.com/kb/HT4228").unwrap(),
-Url::parse("https://support.apple.com/en-ie/guide/safari/sfri11471/mac").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT201265").unwrap(),
-Url::parse("http://itunes.apple.com/us/app/whatsapp-messenger/id310633997?mt=8&fbclid=IwAR30kILqJposAMfPq2JiGQ00o5NwVDc3jvu86wtcmAwzPNCTJXLZaooUpVk").unwrap(),
-Url::parse("https://support.apple.com/downloads/#safari").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/the-identifiers/id564618183").unwrap(),
-Url::parse("https://apps.apple.com/app/packt/id1376577776?ls=1").unwrap(),
-Url::parse("https://appleid.apple.com/auth/authorize?redirect_uri=https%3A%2F%2Fwww.meetup.com%2Fties2%2F&state=returnUri%3Dhttps%253A%252F%252Fwww.meetup.com%252Fhome&client_id=com.meetup.web&response_type=code%20id_token&scope=name%20email&response_mode=form_post").unwrap(),
-Url::parse("https://apps.apple.com/us/app/meetup/id375990038").unwrap(),
-Url::parse("https://apple.com/mac").unwrap(),
-Url::parse("https://apps.apple.com/us/app/tamtam-messenger-video-calls/id1095345669").unwrap(),
-Url::parse("https://apps.apple.com/us/app/nicegram-ai-chat-for-telegram/id1608870673").unwrap(),
-Url::parse("https://apps.apple.com/app/telegram-messenger/id686449807?platform=iphone").unwrap(),
-Url::parse("https://apps.apple.com/us/developer/telegram-fz-llc/id686450210").unwrap(),
-Url::parse("https://support.apple.com/kb/PH2701").unwrap(),
-Url::parse("https://apps.apple.com/us/app/status-saver-video-photo-save/id1528272582").unwrap(),
-Url::parse("https://itunes.apple.com/us/genre/id6005").unwrap(),
-Url::parse("https://apps.apple.com/us/app/telegram-tools/id1621317550").unwrap(),
-Url::parse("https://apps.apple.com/us/app/telegram-messenger/id686449807").unwrap(),
-Url::parse("https://apps.apple.com/us/charts/iphone/social-networking-apps/6005").unwrap(),
-Url::parse("https://apps.apple.com/us/app/viber-messenger-chats-calls/id382617920").unwrap(),
-Url::parse("https://apps.apple.com/app/telegram-messenger/id686449807/www.apple.com/us/shop/goto/exitstore").unwrap(),
-Url::parse("https://apps.apple.com/app/telegram-messenger/id686449807/www.apple.com/us/shop/goto/home").unwrap(),
-Url::parse("https://apps.apple.com/us/app/dual-messenger-plus/id1530021020").unwrap(),
-Url::parse("https://apps.apple.com/app/telegram-messenger/id686449807?platform=ipad").unwrap(),
-Url::parse("https://apps.apple.com/us/app/fandom-videos-news-reviews/id1230063803").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/cold-call-by-harvard-business/id1156646189").unwrap(),
-Url::parse("https://vpp.itunes.apple.com").unwrap(),
-Url::parse("http://itunes.apple.com/us/app/chmplus-chm-reader/id441521818?mt=8").unwrap(),
-Url::parse("http://itunes.apple.com/cn/app/id441756221?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/app/snailgit-free-git-client-with-finder-integration/id1099475282?mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/snailgit/id847260112?l=en&mt=12").unwrap(),
-Url::parse("http://itunes.apple.com/app/ama-connect/id1355068050?uo=5&at=10l9yE").unwrap(),
-Url::parse("http://developer.apple.com/documentation/MacOSX/Conceptual/BPRuntimeConfig/index.html").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT202074").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/pytorch-developer-podcast/id1566080008").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/inaturalist/id421397028?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/app/apple-store/id736535961?pt=2334150&ct=Coursera%20Web%20Promo%20Banner&mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/privacy-management/id1365754887?mt=8").unwrap(),
-Url::parse("https://pcr.apple.com/id1264567358").unwrap(),
-Url::parse("https://support.apple.com/en-is/guide/iphone/iph4f4cbd242/14.0/ios/14.0").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/onlyoffice-documents/id944896972").unwrap(),
-Url::parse("http://support.apple.com/kb/HT1677").unwrap(),
-Url::parse("https://apps.apple.com/app/impact-com/id1629721141").unwrap(),
-Url::parse("https://apps.apple.com/app/id1671420139").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/alchemy/id1503608708").unwrap(),
-Url::parse("https://apps.apple.com/us/app/haaretz-english-edition/id504537897").unwrap(),
-Url::parse("https://apps.apple.com/us/app/microsoft-whiteboard/id1352499399").unwrap(),
-Url::parse("https://support.apple.com/en-gb/guide/apple-business-connect/abcbb749656b/web").unwrap(),
-Url::parse("https://www.apple.com/business/connect").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/id447919187?mt=8").unwrap(),
-Url::parse("https://apps.apple.com/us/app/id447919187?platform=ipad").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/wireguard/id1451685025?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/wireguard/id1441195209?ls=1&mt=8").unwrap(),
-Url::parse("https://apps.apple.com/us/app/geekbench-ml/id1563487010").unwrap(),
-Url::parse("https://apps.apple.com/us/app/geekbench-6/id1565728895").unwrap(),
-Url::parse("https://www.apple.com/wss/fonts?families=SF+Pro,v1|SF+Pro+Icons,v1").unwrap(),
-Url::parse("https://www.apple.com/privacy").unwrap(),
-Url::parse("https://www.apple.com/news").unwrap(),
-Url::parse("https://apps.apple.com/us/app/id1460576753").unwrap(),
-Url::parse("https://apps.apple.com/fr/app/engie-gaz-et-electricit%C3%A9/id1104269908").unwrap(),
-Url::parse("https://apps.apple.com/us/app/bloomberg-business-news/id281941097").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/glassdoor-job-search-salaries/id589698942?mt=8&l=eng").unwrap(),
-Url::parse("https://itunes.apple.com/app/id980888073").unwrap(),
-Url::parse("https://support.apple.com/en-gb/guide/safari/sfri11471/mac").unwrap(),
-Url::parse("https://support.apple.com/en-gb/HT201265").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/99-invisible/id394775318?mt=2").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/99-invisible/id394775318?at=10l9zE").unwrap(),
-Url::parse("https://tv.apple.com/vn/show/acapulco/umc.cmc.8xrkg9zywke7g6a9ahmvpr3l").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2022/04/apple-unveils-the-best-photos-from-the-shot-on-iphone-macro-challenge").unwrap(),
-Url::parse("https://support.apple.com/guide/safari/customize-website-notifications-sfri40734/16.1/mac/13.0").unwrap(),
-Url::parse("http://www.apple.com/safari").unwrap(),
-Url::parse("https://apps.apple.com/app/id1622656471").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/ip2location-geolocation/id771597553?ls=1&mt=8").unwrap(),
-Url::parse("https://apps.apple.com/es/app/diario-abc/id336860348").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=PAMKMDHYQRZYRVK1T2SN&pageLoadId=PBbn8FzEc7Zkrq58&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://apps.apple.com/us/app/nyt-audio/id1549293936").unwrap(),
-Url::parse("https://itunes.apple.com/si/app/mcobiss/id687682312?mt=8").unwrap(),
-Url::parse("https://support.apple.com/en-gb/accessibility").unwrap(),
-Url::parse("https://maps.apple.com/?q=48.8,2.3&t=k").unwrap(),
-Url::parse("https://maps.apple.com/?q=48.8,2.3&t=m").unwrap(),
-Url::parse("https://www.apple.com/legal/privacy/en-ww/cookies").unwrap(),
-Url::parse("https://www.apple.com/legal/privacy/en-ww").unwrap(),
-Url::parse("https://itunes.apple.com/ru/app/id1064581926").unwrap(),
-Url::parse("https://apps.apple.com/app/apple-store/id425194759?pt=417731&ct=enterprise_mobile_comms_LP&mt=8").unwrap(),
-Url::parse("https://apps.apple.com/app/apple-store/id1356686763?pt=417731&ct=lohp&mt=8").unwrap(),
-Url::parse("https://apps.apple.com/us/app/vimeo-create/id1491791513").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/firefox-private-safe-browser/id989804926").unwrap(),
-Url::parse("https://support.apple.com/guide/security/controlling-app-access-to-files-secddd1d86a6/web").unwrap(),
-Url::parse("https://apps.apple.com/us/app/stc-training-club/id575468749").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/what-we-learned-today/id1465904347").unwrap(),
-Url::parse("https://itunes.apple.com/app/apple-store/id1036449551?pt=117888292&ct=discogs-app-page&mt=8").unwrap(),
-Url::parse("https://developer.apple.com/downloads").unwrap(),
-Url::parse("https://developer.apple.com/technologies/tools").unwrap(),
-Url::parse("https://developer.apple.com/macosx/launchd.html").unwrap(),
-Url::parse("https://feedbackassistant.apple.com").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/xcode/id497799835").unwrap(),
-Url::parse("https://developer.apple.com/downloads/index.action").unwrap(),
-Url::parse("https://www.apple.com/itunes").unwrap(),
-Url::parse("https://support.apple.com/downloads#quicktime").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/software-engineering-institute-sei-podcast-series/id566573552?mt=2").unwrap(),
-Url::parse("https://itunes.apple.com/podcast/stephan-livera-podcast/id1415720320?mt=2").unwrap(),
-Url::parse("https://music.apple.com/za/post/1513061930").unwrap(),
-Url::parse("https://support.apple.com/kb/PH17191?locale=it_IT").unwrap(),
-Url::parse("https://apps.apple.com/us/app/the-wall-street-journal/id364387007?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/my-tune-device/id1100377074?ls=1&mt=8").unwrap(),
-Url::parse("https://apps.apple.com/ie/developer/the-irish-times-ltd/id365544597").unwrap(),
-Url::parse("https://itunes.apple.com/us/book/id1457507869").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/cctvnews/id922456579").unwrap(),
-Url::parse("https://support.apple.com/kb/PH21411?locale=en_US").unwrap(),
-Url::parse("https://support.apple.com/kb/PH5042?locale=en_US").unwrap(),
-Url::parse("https://support.apple.com/guide/apple-configurator-2/revive-or-restore-a-mac-with-apple-silicon-apdd5f3c75ad/mac").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/propublica/id355298887?mt=8").unwrap(),
-Url::parse("https://apps.apple.com/app/duckduckgo-privacy-browser/id663592361?ref=spreadprivacy.com").unwrap(),
-Url::parse("https://apps.apple.com/us/app/duckduckgo-privacy-browser/id663592361?ref=spreadprivacy.com").unwrap(),
-Url::parse("https://apps.apple.com/us/app/hubspot/id1107711722").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT204915").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT202303").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/effs-how-to-fix-the-internet/id1539719568").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT201060").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2022/12/apple-advances-user-security-with-powerful-new-data-protections").unwrap(),
-Url::parse("https://developer.apple.com/find-my").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT201088").unwrap(),
-Url::parse("https://apps.apple.com/us/app/remote-desktop-mobile/id714464092").unwrap(),
-Url::parse("https://itunes.apple.com/de/podcast/filterblase-t3n-pioneers-podcast/id1115601393?mt=2").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/an-arm-and-a-leg/id1438778444").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/american-diagnosis-with-dr-celine-gounder/id1282044849?uo=4").unwrap(),
-Url::parse("https://podcasts.apple.com/podcast/where-it-hurts/id1529977695?mt=2").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/what-the-health/id1253607372?mt=2").unwrap(),
-Url::parse("https://apps.apple.com/jp/app/amazon-seller/id794141485").unwrap(),
-Url::parse("https://developer.apple.com/design/tips").unwrap(),
-Url::parse("https://developer.apple.com/library/prerelease/ios//documentation/AVFoundation/Reference/AVCaptureSession_Class/index.html").unwrap(),
-Url::parse("https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/DocumentPickerProgrammingGuide/AccessingDocuments/AccessingDocuments.html").unwrap(),
-Url::parse("https://developer.apple.com/videos/wwdc/2012").unwrap(),
-Url::parse("https://developer.apple.com/videos/play/wwdc2016/413").unwrap(),
-Url::parse("https://developer.apple.com/documentation/appkit/nsstandardkeybindingresponding?language=objc").unwrap(),
-Url::parse("https://developer.apple.com/documentation/foundation/nsdata/3174960-compressed").unwrap(),
-Url::parse("https://support.apple.com/en-gb/guide/preview/prvw587dd90f/mac").unwrap(),
-Url::parse("https://developer.apple.com/reference/xctest").unwrap(),
-Url::parse("https://developer.apple.com/documentation/mapkit/mkgeojsondecoder").unwrap(),
-Url::parse("https://support.apple.com/en-gb/guide/mac-help/mchlp1560/mac").unwrap(),
-Url::parse("https://developer.apple.com/sf-symbols").unwrap(),
-Url::parse("https://support.apple.com/en-gb/guide/mac-help/mchlp1046/mac").unwrap(),
-Url::parse("https://developer.apple.com/documentation/appkit/nsview/1483440-bitmapimagerepforcachingdisplay").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/IDEs/Conceptual/AppDistributionGuide/MaintainingProfiles/MaintainingProfiles.html").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/qa/qa1633/_index.html").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/LanguagesUtilities/Conceptual/iTunesConnect_Guide/Chapters/CreatingiTunesConnectRecord.html").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/userexperience/Conceptual/MobileHIG/index.html#//apple_ref/doc/uid/TP40006556").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/IDEs/Conceptual/AppDistributionGuide/TestingYouriOSApp/TestingYouriOSApp.html#//apple_ref/doc/uid/TP40012582-CH8-SW1").unwrap(),
-Url::parse("https://developer.apple.com/search").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/StringsTextFonts/Conceptual/TextAndWebiPhoneOS/UsingTextClasses/UsingTextClasses.html").unwrap(),
-Url::parse("https://developer.apple.com/library/archive/technotes/tn2339/_index.html").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/UserExperience/Conceptual/MobileHIG/IconMatrix.html").unwrap(),
-Url::parse("https://developer.apple.com/documentation/imageio/3333271-cganimateimageaturlwithblock").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619945-statechangednotification").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/UIKit/Reference/UIActivityViewController_Class").unwrap(),
-Url::parse("https://developer.apple.com/documentation/swift_packages/supportedplatform/3788290-maccatalyst").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/view_controllers/preserving_your_app_s_ui_across_launches").unwrap(),
-Url::parse("https://books.apple.com/us/book/id1551005489").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/StringsTextFonts/Conceptual/TextAndWebiPhoneOS/CustomTextProcessing/CustomTextProcessing.html").unwrap(),
-Url::parse("https://developer.apple.com/download/more").unwrap(),
-Url::parse("https://apps.apple.com/gb/app/scratch/id1671420139").unwrap(),
-Url::parse("https://developer.apple.com/documentation/security/password_autofill/customizing_password_autofill_rules").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/general/conceptual/CocoaEncyclopedia/Introduction/Introduction.html#//apple_ref/doc/uid/TP40010810-CH1-SW1").unwrap(),
-Url::parse("https://itunes.apple.com/gb/course/developing-ios-7-apps-for/id733644550").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/samplecode/PhotoScroller/Introduction/Intro.html").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/general/conceptual/CocoaEncyclopedia/Initialization/Initialization.html#//apple_ref/doc/uid/TP40010810-CH6-SW3").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uicollectionviewcompositionallayout").unwrap(),
-Url::parse("https://developer.apple.com/videos/play/wwdc2015/219").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/General/Conceptual/DevPedia-CocoaCore/AppID.html").unwrap(),
-Url::parse("https://support.apple.com/en-bw/guide/activity-monitor/actmaea30277/mac").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/UserExperience/Conceptual/AutolayoutPG/VisualFormatLanguage.html").unwrap(),
-Url::parse("https://apps.apple.com/app/id1558358855").unwrap(),
-Url::parse("https://apps.apple.com/us/app/rested-simple-http-requests/id421879749").unwrap(),
-Url::parse("https://developer.apple.com/documentation/swiftui/environmentvalues").unwrap(),
-Url::parse("https://developer.apple.com/swift/blog/?id=25").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html#//apple_ref/doc/uid/TP40009249-SW10").unwrap(),
-Url::parse("https://developer.apple.com/documentation/swiftui/state_and_data_flow").unwrap(),
-Url::parse("https://support.apple.com/en-gb/HT206481#thirdparty").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/technotes/tn2259/_index.html").unwrap(),
-Url::parse("https://developer.apple.com/videos/wwdc/2014").unwrap(),
-Url::parse("https://testflight.apple.com").unwrap(),
-Url::parse("https://developer.apple.com/documentation/Charts").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/UserExperience/Conceptual/AutolayoutPG/AdoptingAutoLayout/AdoptingAutoLayout.html").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/Cocoa/Conceptual/ProgrammingWithObjectiveC/WorkingwithBlocks/WorkingwithBlocks.html").unwrap(),
-Url::parse("https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/CodingGuidelines/Articles/NamingMethods.html#//apple_ref/doc/uid/20001282-BCIGIJJF").unwrap(),
-Url::parse("https://apps.apple.com/app/id1578965002").unwrap(),
-Url::parse("https://developer.apple.com/documentation/charts/linemark").unwrap(),
-Url::parse("https://apps.apple.com/app/id1558358855?action=write-review").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/qa/qa1713/_index.html").unwrap(),
-Url::parse("https://www.apple.com/legal/internet-services/itunes/testflight/sren/terms.html").unwrap(),
-Url::parse("https://developer.apple.com/library/mac/releasenotes/ObjectiveC/RN-TransitioningToARC/Introduction/Introduction.html").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/AVFoundation/Reference/AVCaptureVideoPreviewLayer_Class/index.html").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/uncast/id1566634831").unwrap(),
-Url::parse("https://apps.apple.com/us/app/fliff-sports-picks/id1489145500?see-all=reviews").unwrap(),
-Url::parse("https://apps.apple.com/us/app/fliff/id1489145500?ls=1").unwrap(),
-Url::parse("https://support.apple.com/en-gb/HT205280").unwrap(),
-Url::parse("https://developer.apple.com/reference/appkit/nsaccessibility").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/between-the-brackets-a-mediawiki-podcast/id1346062023?mt=2").unwrap(),
-Url::parse("https://support.apple.com/guide/safari/manage-cookies-sfri11471/mac").unwrap(),
-Url::parse("https://developer.apple.com/videos/play/wwdc2023/10121").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/the-stack-overflow-podcast/id1033688462").unwrap(),
-Url::parse("https://apps.apple.com/au/app/amazon-seller/id794141485").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2023/05/apple-launches-20-fun-new-games-for-its-award-winning-apple-arcade-service").unwrap(),
-Url::parse("https://apps.apple.com/us/app/brave-private-web-browser/id1052879175?mt=8&ign-mpt=uo%3D4").unwrap(),
-Url::parse("https://support.apple.com/en-il/guide/safari/manage-cookies-and-website-data-sfri11471/mac").unwrap(),
-Url::parse("https://apps.apple.com/app/google-chrome").unwrap(),
-Url::parse("https://developer.apple.com/documentation/hypervisor").unwrap(),
-Url::parse("https://support.apple.com/guide/safari/manage-cookies-and-website-data-sfri11471/14.0/mac/11.0").unwrap(),
-Url::parse("https://developer.apple.com/reference/foundation/nsurl/1417051-startaccessingsecurityscopedreso?language=objc").unwrap(),
-Url::parse("https://developer.apple.com/documentation/coreservices/1441725-lscopydefaulthandlerforurlscheme?language=objc").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/Foundation/Reference/NSUserActivity_Class/index.html#//apple_ref/occ/instp/NSUserActivity/activityType").unwrap(),
-Url::parse("https://developer.apple.com/documentation/foundation/nsusernotification").unwrap(),
-Url::parse("https://developer.apple.com/design/human-interface-guidelines/notifications").unwrap(),
-Url::parse("https://developer.apple.com/documentation/appkit/nsaboutpaneloptioncredits?language=objc").unwrap(),
-Url::parse("https://developer.apple.com/library/content/documentation/General/Reference/InfoPlistKeyReference/Articles/CocoaKeys.html#//apple_ref/doc/uid/TP40009251-SW35").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/UserExperience/Conceptual/Handoff/HandoffFundamentals/HandoffFundamentals.html").unwrap(),
-Url::parse("https://itunes.apple.com/gb/app/fudget-budget-planner-personal-finance-tracker/id1211707701?mt=12").unwrap(),
-Url::parse("https://developer.apple.com/library/content/documentation/Security/Conceptual/AppSandboxDesignGuide/AppSandboxInDepth/AppSandboxInDepth.html#//apple_ref/doc/uid/TP40011183-CH3-SW16").unwrap(),
-Url::parse("https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/EventOverview/EventArchitecture/EventArchitecture.html#//apple_ref/doc/uid/10000060i-CH3-SW7").unwrap(),
-Url::parse("https://developer.apple.com/documentation/appkit/nsapplication/1428608-servicesmenu?language=objc").unwrap(),
-Url::parse("https://developer.apple.com/documentation/usernotifications/unnotificationresponse").unwrap(),
-Url::parse("https://developer.apple.com/library/archive/technotes/tn2150/_index.html").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html#//apple_ref/doc/uid/TP40009249-102207-TPXREF115").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/general/Reference/InfoPlistKeyReference/Articles/AboutInformationPropertyListFiles.html").unwrap(),
-Url::parse("https://developer.apple.com/reference/appkit/nsapplication/1428479-orderfrontstandardaboutpanelwith?language=objc").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/you-are-not-so-smart-attention/id521594713?i=113727803").unwrap(),
-Url::parse("https://apps.apple.com/us/app/bloomberg-b-unit/id1344778873").unwrap(),
-Url::parse("https://apps.apple.com/app/flat/id1598891661").unwrap(),
-Url::parse("https://support.apple.com/es-es/HT201265").unwrap(),
-Url::parse("https://support.apple.com/kb/PH19214?viewlocale=es_ES&locale=es_ES").unwrap(),
-Url::parse("https://apps.apple.com/us/app/geo-measure-area-calculator/id1099413961").unwrap(),
-Url::parse("https://apps.apple.com/us/app/kml-viewer-and-converter/id1457028727").unwrap(),
-Url::parse("https://developer.apple.com/documentation/safari-release-notes/safari-16_1-release-notes").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/css-tricks-screencasts/id273881728").unwrap(),
-Url::parse("https://developer.apple.com/passkeys").unwrap(),
-Url::parse("https://www.apple.com/accessibility/vision").unwrap(),
-Url::parse("https://podcasts.apple.com/th/podcast/143-fighting-the-climate-crisis-with-data/id1336150688?i=1000618315641").unwrap(),
-Url::parse("https://apps.apple.com/us/app/permanent-archive/id1571883070").unwrap(),
-Url::parse("https://support.apple.com/en-gb/guide/safari/sfri35610/mac").unwrap(),
-Url::parse("https://support.apple.com/HT204416").unwrap(),
-Url::parse("https://itunes.apple.com/fr/app/glassdoor-job-search-salaries/id589698942?mt=8&l=fra").unwrap(),
-Url::parse("https://itunes.apple.com/nl/app/glassdoor-job-search-salaries/id589698942?mt=8&l=nld").unwrap(),
-Url::parse("https://apps.apple.com/cz/app/id766443283?l=cs&mt=8").unwrap(),
-Url::parse("https://support.apple.com/en-euro/guide/safari/sfri11471/mac").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/cdl-test-genie/id1447415467?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/dmv-genius-permit-practice/id513850893?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/app/apple-store/id1145737091").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/famous-birthdays-celebrity/id646707938").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/disconnect-privacy-pro-entire/id1057771839?ls=1&mt=8").unwrap(),
-Url::parse("https://www.apple.com/macbook-air-13-and-15-m2").unwrap(),
-Url::parse("https://www.apple.com/mac-pro").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT210748").unwrap(),
-Url::parse("https://www.apple.com/mac-studio").unwrap(),
-Url::parse("https://support.apple.com/guide/mac-help/calibrate-your-display-mchlp1109/mac").unwrap(),
-Url::parse("https://developer.apple.com/documentation/swiftdata").unwrap(),
-Url::parse("https://apps.apple.com/us/app/uptime-robot-app/id1104878581?utm_source=homepage-section&utm_campaign=website-app-promo&ct=homepage-section-below").unwrap(),
-Url::parse("https://apps.apple.com/us/app/uptime-robot-app/id1104878581?utm_source=homepage-section&utm_campaign=website-app-promo&ct=homepage-section").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/lmax-digital-trading/id1375900638").unwrap(),
-Url::parse("http://www.apple.com/fr/safari").unwrap(),
-Url::parse("https://support.apple.com/guide/safari/manage-cookies-and-website-data-sfri11471").unwrap(),
-Url::parse("https://support.apple.com/da-dk/HT201265").unwrap(),
-Url::parse("https://support.apple.com/da-dk/guide/safari/sfri11471/mac").unwrap(),
-Url::parse("https://developer.apple.com/documentation/quartzcore").unwrap(),
-Url::parse("https://apps.apple.com/mx/app/scotiam%C3%B3vil-mx/id720812080").unwrap(),
-Url::parse("https://developer.apple.com/documentation/virtualization/running_intel_binaries_in_linux_vms_with_rosetta").unwrap(),
-Url::parse("https://apps.apple.com/app/sketch-view-and-mirror/id1609224699").unwrap(),
-Url::parse("http://itunes.apple.com/us/app/faonow/id877618174?ls=1&mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/phys-org/id356443503?mt=8").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/programming-love-with-martin-kleppmann/id1518407590?i=1000495317576").unwrap(),
-Url::parse("https://podcasts.apple.com/ph/podcast/designing-data-intensive-applications-with-martin/id1531450276?i=1000533284011").unwrap(),
-Url::parse("https://apps.apple.com/app/datadog/id1391380318").unwrap(),
-Url::parse("https://help.apple.com/itc/podcasts_connect/#/itcbaf351599").unwrap(),
-Url::parse("https://help.apple.com/itc/podcasts_connect/#/itc2b3780e76").unwrap(),
-Url::parse("https://help.apple.com/itc/podcasts_connect/#/itcb54353390").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT201762").unwrap(),
-Url::parse("https://podcastsconnect.apple.com").unwrap(),
-Url::parse("https://appleid.apple.com/account#!&page=create").unwrap(),
-Url::parse("https://apps.apple.com/app/time.is/id857189816").unwrap(),
-Url::parse("https://podcasts.apple.com/bg/podcast/the-recursive-podcast/id1576538398").unwrap(),
-Url::parse("https://apps.apple.com/jp/app/sail-japanese-conversations/id1434041945").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2020/05/apple-to-host-virtual-worldwide-developers-conference-beginning-june-22").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/true-stories-of-good-people/id1389955443").unwrap(),
-Url::parse("https://support.apple.com/kb/PH21411?locale=nl_NL&viewlocale=en_US").unwrap(),
-Url::parse("https://support.apple.com/kb/PH21411?locale=nl_NL&viewlocale=fr_FR").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/shadertoy/id717961814").unwrap(),
-Url::parse("https://itunes.apple.com/it/app/la-repubblica-mobile/id313084825?mt=8").unwrap(),
-Url::parse("https://apps.apple.com/us/developer/wolfram-group-llc/id715130979").unwrap(),
-Url::parse("https://apps.apple.com/us/app/wolfram-alpha/id548861535").unwrap(),
-Url::parse("https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/PrioritizeWorkAtTheTaskLevel.html").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/jewish-virtual-library/id1336516435?mt=8").unwrap(),
-Url::parse("https://apps.apple.com/ca/app/snikket/id1545164189").unwrap(),
-Url::parse("https://developer.apple.com/wwdc").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/what-the-health/id1253607372?mt=2").unwrap(),
-Url::parse("https://apps.apple.com/us/app/uber/id368677368?uclick_id=c33a8d2c-1390-4ad3-9c63-bb9b1997e716").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/privacy-monitor/id1450531857?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/privacy-monitor/id1451670029?mt=12").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT205223").unwrap(),
-Url::parse("https://opensource.apple.com/source/xnu/xnu-7195.81.3/EXTERNAL_HEADERS/mach-o/fixup-chains.h.auto.html").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/kvikmyndir.is/id1191288272?ls=1&mt=8").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT202663").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/cold-call/id1156646189").unwrap(),
-Url::parse("https://www.apple.com/de/macos").unwrap(),
-Url::parse("https://developer.apple.com/opensource").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2023/06/ios-17-makes-iphone-more-personal-and-intuitive").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2023/06/tvos-17-brings-facetime-and-video-conferencing-to-apple-tv-4k").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2023/06/apple-introduces-the-15-inch-macbook-air").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2023/06/introducing-watchos-10-a-milestone-update-for-apple-watch").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2023/06/ipados-17-brings-new-levels-of-personalization-and-versatility-to-ipad").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2023/06/apple-introduces-m2-ultra").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2023/06/macos-sonoma-brings-new-capabilities-for-elevating-productivity-and-creativity").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT212850").unwrap(),
-Url::parse("https://developer.apple.com/homekit").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2023/06/apple-announces-powerful-new-privacy-and-security-features").unwrap(),
-Url::parse("https://developer.apple.com/videos/play/wwdc2023/10120").unwrap(),
-Url::parse("https://apps.apple.com/app/lmax-global-trading/id884042608").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/damn-interesting/id571835813").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/damn-interesting/id571835813?mt=2").unwrap(),
-Url::parse("https://itunes.apple.com/ro/app/iaBilet/id899564929").unwrap(),
-Url::parse("https://apps.apple.com/app/id6444779277").unwrap(),
-Url::parse("https://podcasts.apple.com/dk/podcast/datatilsynet-podcast-bliv-klogere-p%C3%A5-gdpr/id1481257016").unwrap(),
-Url::parse("https://itunes.apple.com/pl/app/podr%C4%99czny-s%C5%82ownik-hiszpa%C5%84sko-polski-pwn/id1120160080?mt=8&utm_source=ref&utm_medium=web&utm_campaign=6P_0916_ref_201609").unwrap(),
-Url::parse("https://tv.apple.com/movie/evil-dead-rise/umc.cmc.1qv727yx0ev1kdt33soi1xbmy?uo=5&itsct=tv_box_link&itscg=30200&at=1010l369s").unwrap(),
-Url::parse("https://tv.apple.com/movie/spider-man-into-the-spider-verse/umc.cmc.4a2ypxddplsdsahwxrc5fkz4x?uo=5&itsct=tv_box_link&itscg=30200&at=1010l369s").unwrap(),
-Url::parse("https://apps.apple.com/app/id980888073#?platform=iphone").unwrap(),
-Url::parse("https://apps.apple.com/app/id980888073#?platform=mac").unwrap(),
-Url::parse("https://apps.apple.com/gb/app/microsoft-office/id541164041").unwrap(),
-Url::parse("https://apps.apple.com/gb/app/microsoft-teams/id1113153706").unwrap(),
-Url::parse("https://apps.apple.com/us/app/skype-for-iphone/id304878510").unwrap(),
-Url::parse("https://apps.apple.com/us/app/shopify-inbox/id1301681854").unwrap(),
-Url::parse("https://apps.apple.com/us/app/facebook-ads-manager/id964397083").unwrap(),
-Url::parse("https://apps.apple.com/us/app/wix-owner-website-builder/id1545924344").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/xcode/id497799835?mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/wix-com/id1099748482").unwrap(),
-Url::parse("https://apps.apple.com/us/developer/wix-com-inc/id407141669").unwrap(),
-Url::parse("https://apps.apple.com/gb/app/xbox-game-pass/id1374542474").unwrap(),
-Url::parse("https://apps.apple.com/us/app/oculus/id1366478176").unwrap(),
-Url::parse("https://apps.apple.com/ca/app/shop-package-order-tracker/id1223471316").unwrap(),
-Url::parse("https://apps.apple.com/us/app/microsoft-outlook/id951937596").unwrap(),
-Url::parse("https://apps.apple.com/app/facebook/id284882215").unwrap(),
-Url::parse("https://apps.apple.com/us/app/shopify-point-of-sale-pos/id686830644").unwrap(),
-Url::parse("https://apps.apple.com/us/app/dine-by-wix/id1561524113").unwrap(),
-Url::parse("https://apps.apple.com/us/app/shopify-your-ecommerce-store/id371294472").unwrap(),
-Url::parse("https://support.apple.com/fr-afri/guide/safari/sfri11471/mac").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/swann-auction-galleries/id1440439183?ls=1&mt=8").unwrap(),
-Url::parse("http://support.apple.com/kb/DL1251").unwrap(),
-Url::parse("https://support.apple.com/kb/DL1283").unwrap(),
-Url::parse("http://opensource.apple.com/source/IOUSBFamily/IOUSBFamily-458.4.1/AppleUSBOHCI/Classes/AppleUSBOHCI_UIM.cpp").unwrap(),
-Url::parse("https://discussions.apple.com/thread/123054?start=0&tstart=0").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/the-photo-detective/id1255965884?mt=2&mc_cid=67037096ee&mc_eid=[UNIQID]").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/the-photo-detective/id1255965884?mt=2").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/audubon-bird-guide-north-america/id333227386?mt=8&utm_source=bird-guide&utm_medium=national&utm_campaign=iosapp").unwrap(),
-Url::parse("https://apps.apple.com/us/app/clips-clipboard-manager/id1549350243").unwrap(),
-Url::parse("https://store.apple.com/xc/product/MJYT2AM/A").unwrap(),
-Url::parse("https://apps.apple.com/us/app/obscura-3-pro-camera/id1579306989").unwrap(),
-Url::parse("https://apps.apple.com/us/app/halide-mark-ii-pro-camera/id885697368").unwrap(),
-Url::parse("https://developer.apple.com/documentation/xcode-release-notes").unwrap(),
-Url::parse("http://developer.apple.com").unwrap(),
-Url::parse("http://apple.com").unwrap(),
-Url::parse("https://developer.apple.com/library/mac/#documentation/Cocoa/Conceptual/MemoryMgmt/Articles/mmRules.html").unwrap(),
-Url::parse("https://apps.apple.com/us/app/keenetic/id1413650865").unwrap(),
-Url::parse("http://support.apple.com/kb/PH5067").unwrap(),
-Url::parse("https://www.apple.com/privacy/privacy-policy").unwrap(),
-Url::parse("http://developer.apple.com/documentation/Carbon/Conceptual/CoreText_Programming/Overview/chapter_2_section_4.html#//apple_ref/doc/uid/TP40005533-CH3-DontLinkElementID_10").unwrap(),
-Url::parse("http://www.apple.com/macosx/features/bootcamp.html").unwrap(),
-Url::parse("https://apps.apple.com/it/app/vanity-fair-italia/id539933331").unwrap(),
-Url::parse("https://support.apple.com/kb/PH19214").unwrap(),
-Url::parse("https://support.apple.com/en-bh/HT209143").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/appchoices/id894822870").unwrap(),
-Url::parse("https://support.apple.com/safari").unwrap(),
-Url::parse("https://support.apple.com/es-es/safari").unwrap(),
-Url::parse("https://podcasts.apple.com/il/podcast/cpradio/id1491720376").unwrap(),
-Url::parse("https://apps.apple.com/ca/app/thestar-com-iphone/id379481068").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/78-logflare-with-chase-granberry/id1516100616?i=1000545628124").unwrap(),
-Url::parse("https://support.apple.com/en-gb/guide/safari/manage-cookies-and-website-data-sfri11471/mac").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/73-elixir-at-supabase-with-paul-copplestone/id1516100616?i=1000542087899").unwrap(),
-Url::parse("https://developer.apple.com/library/mac/documentation/MobileCoreServices/Reference/UTTypeRef/#//apple_ref/doc/constant_group/UTI_Text_Types").unwrap(),
-Url::parse("https://developer.apple.com/library/prerelease/ios/documentation/Foundation/Reference/NSItemProvider_Class/index.html").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/Foundation/Reference/NSExtensionItem_Class/#//apple_ref/occ/instp/NSExtensionItem/attachments").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/Foundation/Reference/NSExtensionItem_Class").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/Foundation/Reference/NSExtensionContext_Class").unwrap(),
-Url::parse("https://developer.apple.com/library/prerelease/ios/documentation/General/Conceptual/ExtensibilityPG/NotificationCenter.html").unwrap(),
-Url::parse("https://developer.apple.com/videos/play/wwdc2022/110362").unwrap(),
-Url::parse("https://music.apple.com/artist/249646").unwrap(),
-Url::parse("https://developer.apple.com/library/content/documentation/DeveloperTools/Conceptual/MachOTopics/1-Articles/x86_64_code.html").unwrap(),
-Url::parse("https://opensource.apple.com/source/ld64/ld64-123.2.1/src/abstraction/MachOTrie.hpp").unwrap(),
-Url::parse("https://support.apple.com/en-in/guide/safari/sfri11471/mac").unwrap(),
-Url::parse("https://apps.apple.com/app/tmbapp-metro-bus-barcelona/id387847254?pt=216671&ct=web_tmb&mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/it/app/itpc-carabinieri/id858588594?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/tw/app/dramexchange-world-leading/id1080865522?mt=8").unwrap(),
-Url::parse("http://www.apple.com/downloads/macosx/development_tools/svnx.html").unwrap(),
-Url::parse("https://apps.apple.com/ch/app/hostpoint-meet/id1511503042").unwrap(),
-Url::parse("http://www.apple.com/music/store").unwrap(),
-Url::parse("http://support.apple.com/kb/HT4009").unwrap(),
-Url::parse("http://www.apple.com/support/itunes/account").unwrap(),
-Url::parse("http://www.apple.com/support/ipad").unwrap(),
-Url::parse("https://podcasts.apple.com/podcast/id1559261829").unwrap(),
-Url::parse("https://developer.apple.com/library/mac/releasenotes/DeveloperTools/RN-Xcode/Chapters/xc6_release_notes.html#//apple_ref/doc/uid/TP40001051-CH4-SW10").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/unstyled/id1171140955").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/the-cs-ed-podcast/id1486344352").unwrap(),
-Url::parse("https://apps.apple.com/us/app/speechify-text-to-speech-audio/id1209815023").unwrap(),
-Url::parse("https://itunes.apple.com/es/app/marina-dor/id925265167?mt=8").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2020/11/apple-unleashes-m1").unwrap(),
-Url::parse("https://developer.apple.com/news/?id=09032019a").unwrap(),
-Url::parse("https://developer.apple.com/library/archive/documentation/General/Conceptual/ExtensibilityPG/Finder.html").unwrap(),
-Url::parse("https://developer.apple.com/devcenter/download.action?path=/videos/wwdc_2012__hd/session_241__cocoa_interprocess_communication_with_xpc.mov").unwrap(),
-Url::parse("https://opensource.apple.com/source/text_cmds/text_cmds-68/wc/wc.c.auto.html").unwrap(),
-Url::parse("https://support.apple.com/guide/shortcuts-mac/welcome/mac").unwrap(),
-Url::parse("https://developer.apple.com/metal/shader-converter").unwrap(),
-Url::parse("https://support.apple.com/en-ie/HT202074").unwrap(),
-Url::parse("https://apps.apple.com/app/userscripts/id1463298887").unwrap(),
-Url::parse("http://linkmaker.itunes.apple.com").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/smart-author/id1298180300?mt=2").unwrap(),
-Url::parse("https://developer.apple.com/documentation/macos-release-notes/macos-13_4-release-notes").unwrap(),
-Url::parse("https://books.apple.com/book/packaging-for-apple-administrators/id1173928620?mt=11&app=itunes&at=1001lpyX&ct=newsletter").unwrap(),
-Url::parse("https://developer.apple.com/documentation/visionos").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT202491").unwrap(),
-Url::parse("https://geo.itunes.apple.com/us/book/packaging-for-apple-administrators/id1173928620?ls=1&mt=11&at=1001lpyX").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2023/05/app-store-stopped-more-than-2-billion-in-fraudulent-transactions-in-2022").unwrap(),
-Url::parse("https://developer.apple.com/wwdc23/10165").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2023/05/apple-previews-live-speech-personal-voice-and-more-new-accessibility-features").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213544").unwrap(),
-Url::parse("https://developer.apple.com/documentation/Combine").unwrap(),
-Url::parse("https://developer.apple.com/documentation/foundation/userdefaults").unwrap(),
-Url::parse("https://support.apple.com/kb/DL1998").unwrap(),
-Url::parse("https://developer.apple.com/wwdc20/10185").unwrap(),
-Url::parse("https://books.apple.com/book/packaging-for-apple-administrators/id1173928620?mt=11&app=itunes&at=1001lpyX").unwrap(),
-Url::parse("https://developer.apple.com/wwdc21/10095").unwrap(),
-Url::parse("https://developer.apple.com/go/?id=macos-10.15-sdk-rn").unwrap(),
-Url::parse("https://developer.apple.com/documentation/safari-release-notes/safari-16_5-release-notes").unwrap(),
-Url::parse("https://developer.apple.com/documentation/swift/commandline#topics").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT207245").unwrap(),
-Url::parse("https://developer.apple.com/wwdc21/10261").unwrap(),
-Url::parse("https://developer.apple.com/documentation/security/notarizing_macos_software_before_distribution").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT202998").unwrap(),
-Url::parse("https://developer.apple.com/documentation/security/notarizing_macos_software_before_distribution/customizing_the_notarization_workflow").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213327").unwrap(),
-Url::parse("https://developer.apple.com/wwdc23/10053").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213765").unwrap(),
-Url::parse("https://support.apple.com/guide/certifications/welcome/web").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT208891").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT207243").unwrap(),
-Url::parse("https://www.apple.com/ios/ios-17-preview").unwrap(),
-Url::parse("https://developer.apple.com/wwdc23/10002").unwrap(),
-Url::parse("https://geo.itunes.apple.com/us/book/learning-the-bash-shell/id396890728?mt=11&at=1001lpyX").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213760").unwrap(),
-Url::parse("https://developer.apple.com/wwdc23/10263").unwrap(),
-Url::parse("https://developer.apple.com/documentation/SwiftUI").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213250").unwrap(),
-Url::parse("https://support.apple.com/guide/certifications/document-revision-history-apc353b1b736/web").unwrap(),
-Url::parse("https://developer.apple.com/videos/play/wwdc2021/10132").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT207242").unwrap(),
-Url::parse("https://security.apple.com/blog/what-if-we-had-sockpuppet-in-ios16").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT207936").unwrap(),
-Url::parse("https://support.apple.com/guide/deployment/document-revision-history-dep9d814c555/web").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213810").unwrap(),
-Url::parse("https://www.apple.com/ipados/ipados-17-preview").unwrap(),
-Url::parse("https://developer.apple.com/wwdc23/10039").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213812").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213757").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213407").unwrap(),
-Url::parse("https://developer.apple.com/design/human-interface-guidelines/designing-for-visionos").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT207246").unwrap(),
-Url::parse("https://developer.apple.com/documentation/xcode/writing-symbol-documentation-in-your-source-files").unwrap(),
-Url::parse("https://developer.apple.com/wwdc23/10041").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2023/05/apple-launches-new-concert-discovery-features-on-apple-maps-and-apple-music").unwrap(),
-Url::parse("https://developer.apple.com/wwdc23/10040").unwrap(),
-Url::parse("https://developer.apple.com/documentation/foundation/url").unwrap(),
-Url::parse("https://geo.itunes.apple.com/book/property-lists-preferences-profiles-for-apple-administrators/id1213903756?ls=1&mt=11&at=1001lpyX&ct=bookpage").unwrap(),
-Url::parse("https://developer.apple.com/wwdc21/10018").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT207247").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213408").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213758").unwrap(),
-Url::parse("https://developer.apple.com/documentation/xcode-release-notes/xcode-15-release-notes").unwrap(),
-Url::parse("https://books.apple.com/book/terminal-and-shell/id1550617226?mt=11&app=itunes&at=1001lpyX").unwrap(),
-Url::parse("https://developer.apple.com/documentation/technotes/tn3147-migrating-to-the-latest-notarization-tool").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213761").unwrap(),
-Url::parse("https://developer.apple.com/wwdc21/10062").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213813").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213470").unwrap(),
-Url::parse("https://developer.apple.com/documentation/ios-ipados-release-notes/ios-ipados-16_5-release-notes").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT204451").unwrap(),
-Url::parse("https://support.apple.com/guide/tv/whats-new-in-tvos-16-atvb457e13ae/tvos").unwrap(),
-Url::parse("https://www.apple.com/apple-vision-pro").unwrap(),
-Url::parse("https://developer.apple.com/tutorials/swiftui").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213268").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT207244").unwrap(),
-Url::parse("https://developer.apple.com/documentation/swiftui/building-layouts-with-stack-views").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213317").unwrap(),
-Url::parse("https://developer.apple.com/documentation/corefoundation/preferences_utilities").unwrap(),
-Url::parse("https://www.apple.com/watchos/watchos-preview").unwrap(),
-Url::parse("https://developer.apple.com/wwdc23/10120").unwrap(),
-Url::parse("https://developer.apple.com/library/archive/technotes/tn2065/_index.html").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213417").unwrap(),
-Url::parse("https://developer.apple.com/wwdc20/10040").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213759").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213808").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213762").unwrap(),
-Url::parse("https://developer.apple.com/documentation/coreservices/launch_services").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT202999").unwrap(),
-Url::parse("https://developer.apple.com/documentation/appkit/nsworkspace#topics").unwrap(),
-Url::parse("https://books.apple.com/book/moving-to-zsh/id1483591353?mt=11&app=itunes&at=1001lpyX").unwrap(),
-Url::parse("https://support.apple.com/guide/deployment/whats-new-dep950aed53e/web").unwrap(),
-Url::parse("https://support.apple.com/kb/HT213811").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT210060").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT208714").unwrap(),
-Url::parse("https://developer.apple.com/wwdc20/10119").unwrap(),
-Url::parse("https://developer.apple.com/documentation/tvos-release-notes/tvos-16_5-release-notes").unwrap(),
-Url::parse("https://developer.apple.com/documentation/swiftui/appstorage").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213764").unwrap(),
-Url::parse("https://developer.apple.com/bug-reporting").unwrap(),
-Url::parse("https://www.apple.com/macos/sonoma-preview").unwrap(),
-Url::parse("https://developer.apple.com/wwdc23/10254").unwrap(),
-Url::parse("https://books.apple.com/us/book/moving-to-zsh/id1483591353?mt=11&app=itunes&at=1001lpyX").unwrap(),
-Url::parse("https://developer.apple.com/documentation/xcode-release-notes/xcode-14_3_1-release-notes").unwrap(),
-Url::parse("https://support.apple.com/guide/personal-safety/welcome/web").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT201222").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213752").unwrap(),
-Url::parse("https://geo.itunes.apple.com/us/book/bash-cookbook/id396898692?mt=11&at=1001lpyX").unwrap(),
-Url::parse("https://developer.apple.com/wwdc21/10192").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213436").unwrap(),
-Url::parse("https://developer.apple.com/learn/curriculum").unwrap(),
-Url::parse("https://developer.apple.com/wwdc23/102").unwrap(),
-Url::parse("https://geo.itunes.apple.com/nl/book/take-control-mac-command-line-terminal-second-edition/id1234016278?l=en&mt=11&at=1001lpyX").unwrap(),
-Url::parse("https://developer.apple.com/wwdc23/10256").unwrap(),
-Url::parse("https://www.apple.com/newsroom/2023/05/apple-brings-final-cut-pro-and-logic-pro-to-ipad").unwrap(),
-Url::parse("https://geo.itunes.apple.com/book/classic-shell-scripting/id396887958?mt=11&at=1001lpyX").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT201224").unwrap(),
-Url::parse("https://geo.itunes.apple.com/us/book/packaging-for-apple-administrators/id1173928620?ls=1&mt=11&at=1001lpyX&ct=bookpage").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT208050").unwrap(),
-Url::parse("https://books.apple.com/book/moving-to-zsh/id1483591353?mt=11&app=itunes&at=1001lpyX&ct=newsletter").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213809").unwrap(),
-Url::parse("https://developer.apple.com/wwdc23/101").unwrap(),
-Url::parse("https://developer.apple.com/visionos").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT213814").unwrap(),
-Url::parse("https://developer.apple.com/documentation/appkit/nsworkspace").unwrap(),
-Url::parse("https://apps.apple.com/us/app/afends-au/id1568128486").unwrap(),
-Url::parse("https://podcasts.apple.com/ee/podcast/carbon-visibility-in-supply-chains/id1577110956?i=1000597928373").unwrap(),
-Url::parse("https://podcasts.apple.com/ee/podcast/the-what-and-why-of-sustainable-freight/id1577110956?i=1000595798531").unwrap(),
-Url::parse("https://podcasts.apple.com/ee/podcast/how-to-build-a-sustainable-freight-ecosystem/id1577110956?i=1000596983877").unwrap(),
-Url::parse("https://support.apple.com/kb/PH17191").unwrap(),
-Url::parse("https://support.apple.com/es_ES/downloads").unwrap(),
-Url::parse("https://www.apple.com/es/itunes").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/raise-discounted-gift-cards/id922643195?mt=8").unwrap(),
-Url::parse("https://apps.apple.com/us/app/%E6%97%A5%E6%9C%AC%E5%A7%93%E6%B0%8F%E8%AA%9E%E6%BA%90%E8%BE%9E%E5%85%B8-%E3%82%AA%E3%83%95%E3%83%A9%E3%82%A4%E3%83%B3/id1471109324?mt=8").unwrap(),
-Url::parse("https://apps.apple.com/us/app/pocketbook-reader/id805488884").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/the-atlas-obscura-podcast/id1555769970").unwrap(),
-Url::parse("https://apps.apple.com/us/app/microsoft-teams/id1113153706").unwrap(),
-Url::parse("https://apps.apple.com/lt/app/seb-lietuva-m-bankas/id445885936").unwrap(),
-Url::parse("https://apps.apple.com/app/myfootprint-seb/id1589232172").unwrap(),
-Url::parse("https://support.apple.com/en-us/guide/safari/sfri11471/mac").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/m1-finance/id1071915644").unwrap(),
-Url::parse("https://apps.apple.com/us/app/m1-finance/id1071915644").unwrap(),
-Url::parse("https://discussions.apple.com/thread/1107629").unwrap(),
-Url::parse("http://lists.apple.com/archives/Objc-language/2005/Aug/msg00021.html").unwrap(),
-Url::parse("http://lists.apple.com/archives/Objc-language/2005/Aug/threads.html#00019").unwrap(),
-Url::parse("http://developer.apple.com/TOOLS/Xcode").unwrap(),
-Url::parse("https://developer.apple.com/library/content/documentation/Xcode/Reference/xcode_ref-Asset_Catalog_Format/index.html").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/north-star-podcast/id1170914572").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/software-engineering-institute-sei-podcast-series/id566573552?mt=2").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/podcasting-smarter/id566573552?mt=2").unwrap(),
-Url::parse("https://apps.apple.com/us/app/3d%E3%82%B9%E3%82%AD%E3%83%A3%E3%83%8A/id1581786259").unwrap(),
-Url::parse("https://apps.apple.com/app/goodscan-3d/id1600905202").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/voxel-draw/id1084634617").unwrap(),
-Url::parse("https://developer.apple.com/library/archive/documentation/TextFonts/Conceptual/CocoaTextArchitecture/TypoFeatures/TextSystemFeatures.html").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/wikipedia/id324715238").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/graphistania-podcast-for-neo4j-graph-database-community/id975377379").unwrap(),
-Url::parse("https://support.apple.com/zh-cn/guide/iphone/iph10d7bdf26/ios").unwrap(),
-Url::parse("https://apps.apple.com/us/app/wave-invoicing/id881629660").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT212608").unwrap(),
-Url::parse("https://apps.apple.com/us/app/tracker-mood-energy-diary/id1187822249#?platform=messages").unwrap(),
-Url::parse("https://support.apple.com/en-gb/guide/safari/welcome/mac").unwrap(),
-Url::parse("https://www.apple.com/mac-mini").unwrap(),
-Url::parse("https://www.apple.com/macbook-air").unwrap(),
-Url::parse("https://apps.apple.com/us/app/noisli/id862773459").unwrap(),
-Url::parse("http://support.apple.com/downloads").unwrap(),
-Url::parse("http://www.apple.com/support").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/lawyers-guns-money/id1332780055?mt=2").unwrap(),
-Url::parse("https://apps.apple.com/us/app/id1520190857").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/big-ideas-in-app-architecture/id1670998452").unwrap(),
-Url::parse("http://www.apple.com/fr/quicktime/download").unwrap(),
-Url::parse("https://geo.itunes.apple.com/us/app/inquire-by-tamper/id1045268097?mt=8&at=11lmME&ct=tamperio").unwrap(),
-Url::parse("http://www.apple.com/es/privacy/use-of-cookies").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT205280").unwrap(),
-Url::parse("https://itunes.apple.com/app/volumio/id1268256519?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/infinity/id1136944945").unwrap(),
-Url::parse("https://itunes.apple.com/de/institution/universitat-konstanz/id604733564").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/cppcast/id968703120").unwrap(),
-Url::parse("https://itunes.apple.com/ie/app/houses-of-the-oireachtas/id783571697?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/app/payoneer/id663338402").unwrap(),
-Url::parse("https://apps.apple.com/us/app/mayo-clinic/id523220194").unwrap(),
-Url::parse("https://apps.apple.com/de/app/stuttgarter-zeitung-app/id517190273").unwrap(),
-Url::parse("https://apps.apple.com/de/app/stuttgarter-zeitung-de/id397131360").unwrap(),
-Url::parse("https://itunes.apple.com/app/depositphotos/id919098408").unwrap(),
-Url::parse("http://www.apple.com/macosx/lion").unwrap(),
-Url::parse("http://www.apple.com/ipad/#smart-cover").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/seek-by-inaturalist/id1353224144").unwrap(),
-Url::parse("https://www.apple.com/maps").unwrap(),
-Url::parse("https://itunes.apple.com/app/apple-store/id982107779").unwrap(),
-Url::parse("https://developer.apple.com/app-sandboxing").unwrap(),
-Url::parse("https://developer.apple.com/library/mac/documentation/UserExperience/Conceptual/AppleHIGuidelines/IconsImages/IconsImages.html#//apple_ref/doc/uid/20000967-TP6").unwrap(),
-Url::parse("https://developer.apple.com/appstore/mac/resources/approval/guidelines.html").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/the-tim-ferriss-show/id863897795").unwrap(),
-Url::parse("https://apps.apple.com/us/app/jetpack-website-builder/id1565481562").unwrap(),
-Url::parse("https://apps.apple.com/ru/app/fandom-videos-news-reviews/id1230063803").unwrap(),
-Url::parse("https://apps.apple.com/us/app/refinitiv-workspace/id1481442629").unwrap(),
-Url::parse("https://itunes.apple.com/app/aliexpress-for-ipad/id854356557").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/aliexpress/id436672029").unwrap(),
-Url::parse("https://apps.apple.com/jp/app/medaca-%E8%87%AA%E5%88%86%E3%81%AE%E5%81%A5%E5%BA%B7%E3%82%92%E5%8F%8E%E7%B4%8D%E3%81%99%E3%82%8B%E3%82%A2%E3%83%97%E3%83%AA/id1190040349?l=en").unwrap(),
-Url::parse("https://machinelearning.apple.com/updates/apple-scholars-aiml-2022").unwrap(),
-Url::parse("https://apps.apple.com/us/app/microscope-journal/id507267328").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/ardan-labs-podcast/id1539547911").unwrap(),
-Url::parse("https://podcasts.apple.com/gb/podcast/thoughtworks-technology-podcast/id881136697?i=1000584890418").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/adweek-magazine/id570229326?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/edsurge-on-air/id972239500#").unwrap(),
-Url::parse("https://apps.apple.com/de/app/bvg-fahrinfo-plus-berlin/id284971745").unwrap(),
-Url::parse("https://apps.apple.com/de/app/taxi-berlin/id443169338").unwrap(),
-Url::parse("https://apps.apple.com/de/app/going-local-berlin/id964438630").unwrap(),
-Url::parse("https://apps.apple.com/app/wearedevs-world-congress-22/id1625710834").unwrap(),
-Url::parse("https://apps.apple.com/us/app/about-berlin/id1425152714").unwrap(),
-Url::parse("https://geo.itunes.apple.com/us/app/serenity-relaxing-ambient/id1185371817?mt=12&at=1010ltPG").unwrap(),
-Url::parse("http://itunes.apple.com/app/id1238649962").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT210380").unwrap(),
-Url::parse("https://www.apple.com/am/mac/m1").unwrap(),
-Url::parse("https://apps.apple.com/us/app/day-one-2-journal-notes/id1044867788").unwrap(),
-Url::parse("https://apps.apple.com/pt/app/go-i-pace/id1397794566").unwrap(),
-Url::parse("https://itunes.apple.com/gb/podcast/nature-podcast/id81934659?mt=2").unwrap(),
-Url::parse("https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6AATIntro.html").unwrap(),
-Url::parse("https://support.apple.com/kb/ph21411?locale=en_US").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/lenovo-3d-catalog/id1164430025?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/app/id1290128888").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/wrike/id890048871?mt=8").unwrap(),
-Url::parse("https://music.apple.com/gb/playlist/bbc-philharmonic-orchestra-of-the-year-2023-nominee/pl.5921732a341d42c99337a1a7cde3f5dd?at=1l3v8Ma&ct=playlist_bbc-philharmonic-orchestra-of-t&itscg=30200&itsct=toolbox_linkbuilder&ls=1&app=music").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=H5FWFAEZ86VNJR9Y25GG&pageLoadId=AqScAGXlLsH2Og2q&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=KGS4XCDCERWHBX904KTF&pageLoadId=BIjEUvwHhteyozzz&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=BG1JH1KFKCF86TW7K07Q&pageLoadId=48P5R1nie5laYcsR&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=XA9CGTJQQQ1532V31779&pageLoadId=FRA8JGqyFpDrcSTk&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=W8PGKFYPV3N1DPJRBSF4&pageLoadId=vhYn8vZlccfbXKXb&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=Y5T2MD9Q99EX901MRJ99&pageLoadId=teMLdIeMi7gtY2D0&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=6QGCKS6Q14B2570A9C6Y&pageLoadId=UadY8Hi2MO0XVEmH&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=20XMEQKV4KM399Z3DJZB&pageLoadId=BN9m4ApVmjfeH44O&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=3JMMYVAR6GKJXFXGDP27&pageLoadId=9SJJwwuJOBhbhzfI&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=RZMTNTNP5ES90TBZSYR8&pageLoadId=KCKIJ7L0Eyu0gPyk&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=TN13W97JDJ647QABA4EX&pageLoadId=q0MJhs7b2apczQAB&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=VKAMXR50EWHMB4JY77AC&pageLoadId=Acc4IyWt1DLl5QfM&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=5N50GSZX2S39GSVMR9PJ&pageLoadId=Zn07nkCtOGonOUfH&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=M5K4BK2KPTB7HA82T3N3&pageLoadId=sDYj4djOnGzMLb7j&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=EJB2JSMXK65J3RYWXPFB&pageLoadId=STrIRrqqustEa93k&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=PZYKXDK1QVNSY5FYGFPR&pageLoadId=tCq5Buyk860Wsvq7&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=J2CN9MEFXKCZJE5CNG1B&pageLoadId=Asy3WiZ6T3IJjEsu&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=ZCVE1TKH3CXEMWANVDXA&pageLoadId=Jaz5nrhH7zBNm33X&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=FEHE4CZJZFY4RKGZPGRS&pageLoadId=HY1aRBlNXutNeFGb&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=1V15C26X8B8C9EVPFRE5&pageLoadId=Lcf8tcXGjBQBFT8r&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=KZP2G549BDRS8DGVZVBN&pageLoadId=CqpUiRnw3DAcgm2V&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=NV4XYEXKB796YX2V0KWK&pageLoadId=6zsVgP3A475l073h&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=WN2BSG3GMWR2FGJTD790&pageLoadId=ELScE1ACkLd2oUS6&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=SSFF13ZDNNGKXC5NK3KE&pageLoadId=PgI0C7FfVLAfdQOi&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=df3314c2-582c-4e54-8eed-d5a4bf97c8ca&pf_rd_r=VYNYGWWXA5TD0JZE3812&pageLoadId=t0yablgDG2JNlBcm&creativeId=9adc166d-b28f-4dd5-bfce-d65bdc6a1a13").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=R1TF8MVA6Z9BHFK6HNSF&pageLoadId=JTgn1d4Qj3BTSPJj&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=KAC02X5CEXSN3T6FJ5W3&pageLoadId=pVNbYTHcSRmy9LsU&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=RMA8JG7DQF12C0ARS2E7&pageLoadId=PEZTCuR49ThJaYNh&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=DFCMWRZ7V68X5FN7JGRM&pageLoadId=1QgdRJO0F04rBxB4&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=165TX149S4EG0GDBJ0M9&pageLoadId=hWqe1UDMwDsii6W7&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=HEVQ06X0PSE73XJJVBFH&pageLoadId=uuXTam0qvCOtsPQF&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=366FZTR6NY1RPVNBENZE&pageLoadId=MkwORBurjRrBD971&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=NMMNFBVFCY6B34C0HNXW&pageLoadId=3PdqbgS3EdEUUJSU&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=1R524XYNPHAKM5B77PXD&pageLoadId=0vie0Jfx9N5FAFbL&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=HB0Z73RFR7W8QTYG5G9S&pageLoadId=4ZOnh5ioc3Waw1FS&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=15fe8fa9-606b-4ab6-be26-fffbd045d7bd&pf_rd_r=RPME3P7RF5N9N0FMGS3W&pageLoadId=erNIxgdVCZ96jQN0&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=8DSBE2NQEEEBS04BDYJN&pageLoadId=5RiIY5ACXO41alrU&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=F25TQKQWG9D0FH2Y7FXN&pageLoadId=ocGVVaBWzhoZjmg8&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=WTCRF4CKKXB1X7KPYCVA&pageLoadId=hOhQ65k2vK0d9SIN&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=HBGSB527KB2RZJ1WT5P1&pageLoadId=L2qQrbXVZkyUqr1d&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=AXYTRZA4PA7VNVSZGGGE&pageLoadId=yVy7b49EZPYG1Oij&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=X0EVR7SXRTDK67WNKX64&pageLoadId=PHHJzGzcBpcV3QiQ&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=DJH9066ABTZD7R1NDM2T&pageLoadId=BvMfERP3XO5ytOIi&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=W0NNQX1JYRFCARHHTVDH&pageLoadId=YLi76rAZB9d7uphe&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=34N4KGM51P71DRKQ19P7&pageLoadId=l5IbeFfrkFLvNVf4&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=85RG7R5NG853KQ8TV9HQ&pageLoadId=oR5sHmKaZlpYTxiR&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=V57XMRPA5VY6AXP2TFSG&pageLoadId=PmuPbx6SlMFSXPwv&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
-Url::parse("https://developer.apple.com/streaming").unwrap(),
-Url::parse("https://developer.apple.com/documentation/http_live_streaming/example_playlists_for_http_live_streaming/creating_a_multivariant_playlist").unwrap(),
-Url::parse("https://itunes.apple.com/es/app/paradores-hoteles-restaurantes/id960788695").unwrap(),
-Url::parse("https://apps.apple.com/us/app/rise-sleep-energy-tracker/id1453884781").unwrap(),
-Url::parse("https://apps.apple.com/us/app/rise-track-sleep-circadian/id1453884781").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/where-we-go-next/id1527944755").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/the-ai-podcast/id1186480811?mt=2&adbsc=social_20161220_68874946&adbid=811257941365882882&adbpl=tw&adbpr=61559439").unwrap(),
-Url::parse("https://developer.apple.com/augmented-reality").unwrap(),
-Url::parse("http://itunes.apple.com/us/app/wikihow-how-to-diy-survival/id309209200?mt=8").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT206481").unwrap(),
-Url::parse("https://developer.apple.com/support/code-signing").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/view_controllers/adding_a_document_browser_to_your_app/presenting_selected_documents").unwrap(),
-Url::parse("https://developer.apple.com/ios/human-interface-guidelines/system-capabilities/quick-look").unwrap(),
-Url::parse("https://developer.apple.com/xcode/swiftui").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uidocumentbrowserviewcontrollerdelegate/2874187-documentbrowser").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uidocumentbrowserviewcontroller").unwrap(),
-Url::parse("https://developer.apple.com/documentation/fileprovider").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619978-contents").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/view_controllers/building_a_document_browser_based_app").unwrap(),
-Url::parse("https://developer.apple.com/library/content/documentation/Miscellaneous/Reference/UTIRef/Articles/System-DeclaredUniformTypeIdentifiers.html").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/airstitch/id645714375?mt=8").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uidocument").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619988-save").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619979-init").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619992-filetype").unwrap(),
-Url::parse("https://developer.apple.com/library/content/documentation/General/Reference/InfoPlistKeyReference/Articles/iPhoneOSKeys.html#//apple_ref/doc/uid/TP40009252-SW37").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619982-documentstate").unwrap(),
-Url::parse("https://www.apple.com/macos/catalina-preview").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/view_controllers/adding_a_document_browser_to_your_app").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619971-load").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uiviewcontroller/1621380-present").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619977-open").unwrap(),
-Url::parse("https://developer.apple.com/library/content/documentation/General/Reference/InfoPlistKeyReference/Articles/iPhoneOSKeys.html#//apple_ref/doc/uid/TP40009252-SW20").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uidocumentbrowserviewcontrollerdelegate").unwrap(),
-Url::parse("https://apps.apple.com/us/app/c-by-ge/id1082828238?ls=1").unwrap(),
-Url::parse("https://music.apple.com/us/artist/dawn-margie-beaton/414795545").unwrap(),
-Url::parse("http://itunes.apple.com/de/app/openseamap/id495210783").unwrap(),
-Url::parse("http://itunes.apple.com/de/app/openseamap/id495210783?mt=8").unwrap(),
-Url::parse("https://apps.apple.com/us/app/3cx-webmeeting/id1039756959").unwrap(),
-Url::parse("https://apps.apple.com/us/app/3cxphone-15-for-3cx-phone/id992045982?ls=1").unwrap(),
-Url::parse("https://apps.apple.com/us/app/3cx/id992045982").unwrap(),
-Url::parse("https://apps.apple.com/us/app/mot%C3%AE/id1046122820?platform=iphone").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/involve-ai-bolstering-customer-success-with-ai-with/id1517181598?i=1000556587495").unwrap(),
-Url::parse("https://apps.apple.com/us/app/work-mobile/id1127865121").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/gopayment/id324389392?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/smart-id/id1188304729?l=et&ls=1&mt=8").unwrap(),
-Url::parse("https://www.apple.com/ios").unwrap(),
-Url::parse("https://support.apple.com/kb/PH17191?locale=es_ES").unwrap(),
-Url::parse("https://itunes.apple.com/app/apple-store/id473512078?pt=356848&at=1l3vpXa&ct=hp&mt=8").unwrap(),
-Url::parse("https://support.apple.com/en-il/guide/safari/sfri11471/mac").unwrap(),
-Url::parse("https://support.apple.com/en-gb/guide/voiceover/vo14111/10/mac/12.0").unwrap(),
-Url::parse("https://support.apple.com/en-gb/guide/iphone/iph3e2e2281/ios").unwrap(),
-Url::parse("https://support.apple.com/en-gb/HT210417").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT206175").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/clustered-conversations-by-weka/id1621362291").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/the-gamesindustry-biz-podcast/id1195829295").unwrap(),
-Url::parse("http://www.apple.com/quicktime/download").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/distributed-with-matt-mullenweg/id1463243282").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/airvisual-real-time-forecast/id1048912974?ls=1&mt=8%27").unwrap(),
-Url::parse("https://itunes.apple.com/gb/app/degiro-beleggen/id1099174456?l=en&mt=8").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/episode-123-dr-martin-weihrauch-smart-in-media-digital/id1490210201?i=1000576089087").unwrap(),
-Url::parse("https://developer.apple.com/documentation/webkit/wkwebview").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/filmstemmer/id1682882422").unwrap(),
-Url::parse("https://apps.apple.com/app/id766443283?mt=8").unwrap(),
-Url::parse("https://apps.apple.com/us/app/tableau-mobile/id434633927").unwrap(),
-Url::parse("https://apps.apple.com/us/app/id1517276406").unwrap(),
-Url::parse("https://apps.apple.com/ch/app/sbb-mobile/id294855237").unwrap(),
-Url::parse("https://apps.apple.com/gb/app/eyeora-vr/id1501679625").unwrap(),
-Url::parse("https://podcasts.apple.com/de/podcast/t3n-catch-up/id1627024991").unwrap(),
-Url::parse("https://podcasts.apple.com/de/podcast/t3n-podcast-das-w%C3%B6chentliche-update-f%C3%BCr-digitale-pioniere/id1115601393").unwrap(),
-Url::parse("https://podcasts.apple.com/de/podcast/t3n-daily/id1643152920").unwrap(),
-Url::parse("https://developer.apple.com/forums/thread/650026").unwrap(),
-Url::parse("https://apps.apple.com/nl/app/wandelnet-nl-wandelroutes/id1463429180").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/it-happened-in-hollywood/id1437795866").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/richard-obrien-the-rocky-horror-picture-show/id1437795866?i=1000617840833").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/between-the-brackets-a-mediawiki-podcast/id1346062023?ls=1").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/the-agile-uprising-podcast/id1163230424?ls=1").unwrap(),
-Url::parse("http://itunes.apple.com/us/podcast/edward-tufte-art-and-science/id297775181").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/code-ish/id1457879893").unwrap(),
-Url::parse("https://apps.apple.com/us/app/google-adsense/id680739529").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/alphabet-scoop/id1359922601?mt=2").unwrap(),
-Url::parse("https://itunes.apple.com/app/apple-store/id626148774?pt=2050088&ct=Modal&mt=8").unwrap(),
-Url::parse("http://opensource.apple.com/source/dyld/dyld-210.2.3/src/dyld_stub_binder.s").unwrap(),
-Url::parse("https://itunes.apple.com/ca/app/bc-lions-exp/id1263298513?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/es/app/telegram-messenger/id686449807?mt=8").unwrap(),
-Url::parse("https://developer.apple.com/documentation/security/certificate_key_and_trust_services/trust?language=objc").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT210176").unwrap(),
-Url::parse("https://developer.apple.com/documentation/security/secure_transport").unwrap(),
-Url::parse("https://developer.apple.com/library/archive/documentation/Darwin/Conceptual/KernelProgramming/BSD/BSD.html#//apple_ref/doc/uid/TP30000905-CH214-TPXREF101").unwrap(),
-Url::parse("https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/getsockopt.2.html").unwrap(),
-Url::parse("https://apps.apple.com/us/app/wallmine-stocks-bitcoin/id1458107499").unwrap(),
-Url::parse("https://apps.apple.com/us/app/dfds-ferries-cruises/id1447470946").unwrap(),
-Url::parse("https://itunes.apple.com/in/app/valley-air-district/id526195423").unwrap(),
-Url::parse("https://support.apple.com/en-gb/HT211094").unwrap(),
-Url::parse("https://itunes.apple.com/app/apple-store/id954157737?pt=49113808&ct=georgegarside.com%2Fapps&mt=8&at=1001lwpF").unwrap(),
-Url::parse("https://itunes.apple.com/app/apple-store/id989353027?pt=49113808&ct=georgegarside.com%2Fapps&mt=8&at=1001lwpF").unwrap(),
-Url::parse("http://www.apple.com/trailers/fox/thehappening").unwrap(),
-Url::parse("http://itunes.apple.com/gb/podcast/british-library-podcasts/id174455610?ign-mpt=uo=4").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/apple-store/id535886823?pt").unwrap(),
-Url::parse("https://support.apple.com/it-it/guide/safari/sfri11471/mac").unwrap(),
-Url::parse("https://support.apple.com/en-ca/guide/safari/sfri11471/mac").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/foodist-with-darya-rose-ph.d/id1090954776?mt=2").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/foodist-with-darya-rose-ph-d/id1090954776").unwrap(),
-Url::parse("https://developer.apple.com/documentation/app_clips/creating_an_app_clip").unwrap(),
-Url::parse("https://developer.apple.com/documentation/bundleresources/information_property_list/nslocationwheninuseusagedescription").unwrap(),
-Url::parse("https://developer.apple.com/design/human-interface-guidelines/ios/system-capabilities/notifications").unwrap(),
-Url::parse("https://developer.apple.com/documentation/corelocation/choosing_the_authorization_level_for_location_services").unwrap(),
-Url::parse("https://developer.apple.com/documentation/usernotifications/unnotificationserviceextension").unwrap(),
-Url::parse("https://developer.apple.com/documentation/uikit/uiapplicationdelegate/1623013-application").unwrap(),
-Url::parse("https://developer.apple.com/documentation/app_clips/enabling_notifications_in_app_clips").unwrap(),
-Url::parse("https://developer.apple.com/library/archive/qa/qa1633/_index.html").unwrap(),
-Url::parse("https://developer.apple.com/documentation/bundleresources/information_property_list/nslocationalwaysusagedescription").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT203609").unwrap(),
-Url::parse("https://developer.apple.com/documentation/usernotifications/asking_permission_to_use_notifications").unwrap(),
-Url::parse("https://itunes.apple.com/ch/app/india-air-quality/id988591878?l=en&mt=8").unwrap(),
-Url::parse("https://developer.apple.com/documentation/widgetkit/timelineprovider").unwrap(),
-Url::parse("https://developer.apple.com/videos/play/wwdc2020/10034").unwrap(),
-Url::parse("https://developer.apple.com/documentation/widgetkit/creating-a-widget-extension").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT208309").unwrap(),
-Url::parse("https://support.apple.com/guide/shortcuts/add-import-questions-before-sharing-apdf330fd3a0/ios").unwrap(),
-Url::parse("https://support.apple.com/guide/shortcuts/welcome/ios").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/google-photos-store-search/id962194608?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/app/id853371601").unwrap(),
-Url::parse("https://developer.apple.com/technologies/ios").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/google-translate/id414706506?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/google-slides/id879478102?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/youtube/id544007664?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/google-docs/id842842640?mt=8").unwrap(),
-Url::parse("http://developer.apple.com/iphone/index.action").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/google-sheets/id842849113?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/app/idid909319292").unwrap(),
-Url::parse("https://itunes.apple.com/app/id905060486").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/google-play-music/id691797987?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/google-calendar/id909319292?mt=8").unwrap(),
-Url::parse("http://developer.apple.com/opensource").unwrap(),
-Url::parse("https://developer.apple.com/library/ios/documentation/CoreBluetooth/Reference/CBCentralManager_Class/index.html#//apple_ref/occ/instm/CBCentralManager/cancelPeripheralConnection:").unwrap(),
-Url::parse("https://developer.apple.com/library/prerelease/ios/documentation/CoreBluetooth/Reference/CBCentralManager_Class/index.html#//apple_ref/doc/constant_group/Central_Manager_Initialization_Options").unwrap(),
-Url::parse("https://developer.apple.com/library/prerelease/ios/documentation/CoreBluetooth/Reference/CBPeripheral_Class/index.html#//apple_ref/c/tdef/CBCharacteristicWriteType").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/brian-collins/id328074695?i=1000384718204&mt=2").unwrap(),
-Url::parse("https://apps.apple.com/us/app/syncfusion-succinctly-series/id1336430099").unwrap(),
-Url::parse("https://www.apple.com/uk/swift").unwrap(),
-Url::parse("https://apps.apple.com/nz/app/1-news/id329786380").unwrap(),
-Url::parse("https://apps.apple.com/app/apple-store/id1081530898").unwrap(),
-Url::parse("https://apps.apple.com/us/app/the-new-yorker-today/id1081530898").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT202039").unwrap(),
-Url::parse("https://apps.apple.com/us/app/flipboard-latest-stories").unwrap(),
-Url::parse("https://apps.apple.com/us/app/the-new-yorker/id1081530898").unwrap(),
-Url::parse("https://apps.apple.com/be/app/go-i-pace/id1397794566?l=fr").unwrap(),
-Url::parse("https://apps.apple.com/be/app/go-i-pace/id1397794566?l=nl").unwrap(),
-Url::parse("https://apps.apple.com/app/apple-store/id1241040030?pt=64957&ct=blizzard-global-footer&mt=8").unwrap(),
-Url::parse("http://www.apple.com/macosx").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/mht-to-webarchive/id838885505?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/css-tidy/id700229660?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/tcl-runner/id701474998?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/python-lint/id891162632?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/python-runner/id689176796?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/php-lint/id878546858?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/web-proxy/id721103033?mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/huo-che-ding-piao-zhu-shou/id494486644?ls=1&mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/socks-proxy/id1116965059?mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/set-file-icon/id787613939?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/copy-as-text/id869232291?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/html-tidy/id700229601?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/php-runner/id686855602?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/chinese-conversion/id720125982?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/no-trash-delete-files-without/id777213066?ls=1&mt=12").unwrap(),
-Url::parse("https://apps.apple.com/app/new-file-menu/id1064959555?l=en&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/webarchive-to-mht/id838885411?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/ip-messenger/id875939790?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/chmplus-chm-reader/id588628901?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/jslint/id757770685?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/simple-svn-server/id1040404148?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/epub-reader/id591299853?ls=1&mt=12").unwrap(),
-Url::parse("https://apps.apple.com/cn/story/id1520078797?l=en").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/copy-as-html/id868049821?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/simple-hg-server/id1040403596?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/svn-status/id842480930?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/extractor-extract-rar-zip/id747644886?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/css-beautifier/id757811119?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/monitor-test/id904107000?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/favicon-creator/id787259510?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/simple-git-server/id1040403936?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/mht-viewer/id622609427?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/app/context-menu/id1236813619?l=en&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/copy-file-path/id777213048?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/hg-status/id993527882?mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/shell-checker/id892605935?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/paste-as-file/id825675368?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/archive-viewer/id598123939?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/dock-icons/id788702540?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/json-lint/id894042665?ls=1&mt=12").unwrap(),
-Url::parse("https://support.apple.com/kb/ht4094").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/lua-runner/id689181261?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/icon-extractor/id803083078?ls=1&mt=12").unwrap(),
-Url::parse("https://apps.apple.com/app/context-menu/id1236813619?l=en&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/snailfossil/id1226366997?ls=1&mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/webarchive-to-html/id844587292?ls=1&mt=12").unwrap(),
-Url::parse("https://apps.apple.com/app/git-credential-manager/id1575689398?l=en&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/node-runner/id689175950?ls=1&mt=12").unwrap(),
-Url::parse("https://apps.apple.com/app/java-runner/id689184215").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/perl-runner/id689178925?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/icon-converter/id803070906?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/html5-tidy/id700229688?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/startup-time/id875911449?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/app/chmplus-pro-chm-reader/id441521818?mt=8").unwrap(),
-Url::parse("https://apps.apple.com/app/filedrop-quick-file-transfer/id1605772776?l=en&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/app/webdav-server/id1185282799?l=en&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/git-status/id842481797?mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/new-file-here/id777221999?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/app/new-file-menu-free/id1066302071?l=en&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/chm-to-html/id869362084?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/share-clipboard/id619587463?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/ipagerank-pagerank-checker/id433671401?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/ruby-runner/id689179916?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/js-runner/id689183220?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/js-beautifier/id700229527?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/source-code-counter/id889599066?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/mht-to-html/id844591996?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/clipboard-viewer/id870366108?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/qr-clipboard/id869183896?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/chm-lite/id443454952?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/cn/artist/langui.net/id433671404?l=en&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/app/new-file-menu/id1064959555?l=en&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/snailhg/id1006394020?mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/app/chmplus-lite-chm-reader/id443454952?mt=8").unwrap(),
-Url::parse("https://apps.apple.com/app/isvn/id861041982?ls=1").unwrap(),
-Url::parse("https://apps.apple.com/app/ftp-mounter/id1437045769?l=en&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/hg-server/id1034692383?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/git-server/id959332098?ls=1&mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/cn/app/huo-che-ding-piao-zhu-shou/id494486644?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/app/chmplus-chm-reader/id441521818?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/network-speed-status/id875176096?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/chmplus-free-chm-reader/id554596300?ls=1&mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/app/zipmounter/id1315374401?l=en&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/smartcopy/id884734329?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/svn-server/id685191052?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/sftp-server/id1300094971?ls=1&mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/css-lint/id757865011?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/new-file-menu/id1064959555?ls=1&mt=12").unwrap(),
-Url::parse("https://itunes.apple.com/nl/app/logius/id1167926346?ls=1&mt=8").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/e356-jeff-seibert-co-founder-at-digits/id1442272943?i=1000567828292").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/episode-2-fintech-what-the-heck-digits/id1578932483?i=1000568891111").unwrap(),
-Url::parse("https://podcasts.apple.com/us/podcast/digitalconomics-podcast/id1611260822").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/ticketmaster/id500003565?mt=8").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/bitfinex/id1436383182?ls=1&mt=8").unwrap(),
-Url::parse("http://support.apple.com/kb/PH17191?viewlocale=fr_FR").unwrap(),
-Url::parse("https://support.apple.com/ja-jp/HT202074").unwrap(),
-Url::parse("https://apps.apple.com/app/apple-store/id1625333066?pt=125060841&ct=website-main&mt=8").unwrap(),
-Url::parse("https://apps.apple.com/app/id1594445686").unwrap(),
-Url::parse("https://itunes.apple.com/app/id911410294").unwrap(),
-Url::parse("https://apps.apple.com/app/realplayer-mobile/id1578950397").unwrap(),
-Url::parse("https://apps.apple.com/us/app/trnio-plus-3d-scanner/id1546160586").unwrap(),
-Url::parse("https://apps.apple.com/us/app/trnio-plus-3d-scanner/id1546160586?itsct=apps_box_badge&itscg=30200").unwrap(),
-Url::parse("https://apps.apple.com/pt/app/3cx/id992045982").unwrap(),
-Url::parse("https://itunes.apple.com/gb/podcast/art-matters/id1343288276?mt=2").unwrap(),
-Url::parse("https://itunes.apple.com/cn/podcast/%E6%9E%81%E5%AE%A2%E7%94%B5%E5%8F%B0/id914427651?mt=2").unwrap(),
-Url::parse("https://support.apple.com/en-us/HT201859").unwrap(),
-Url::parse("https://itunes.apple.com/us/podcast/the-big-picture/id1439252196?mt=2").unwrap(),
-Url::parse("https://itunes.apple.com/us/app/uberfit-runner/id1033916715?mt=8&at=10lwCW&uo=8&at=10lwCW").unwrap(),
-Url::parse("https://www.apple.com/apple-music").unwrap(),
-Url::parse("https://www.apple.com/legal/internet-services/itunes/us/terms.html").unwrap(),
- ];
+ let mut rng = rand::thread_rng();
+ for _ in 0..rng.gen_range(1..100) {
+ path.push(rng.gen_range(b'a'..=b'z') as char);
+ }
- // pre-initialize domain prefix list
- letsencrypt_urls[0].domain();
+ Url::parse(&format!("https://{}/{}", domain.as_str(), path)).unwrap()
+}
- let letsencrypt_url_responses: Vec<UrlResponse> = letsencrypt_urls
- .clone()
- .into_iter()
- .map(|url| UrlResponse::Success { url })
- .collect();
+fn random_responses(num: usize) -> Vec<JobResponse> {
+ let mut responses = Vec::with_capacity(num);
+ for _ in 0..num {
+ let domain = rand_domain();
- let apple_url_responses: Vec<UrlResponse> = apple_urls
- .clone()
- .into_iter()
- .map(|url| UrlResponse::Success { url })
- .collect();
+ let url_responses = vec![
+ UrlResponse::Success {
+ url: rand_url(&domain)
+ };
+ 100
+ ];
- let responses = vec![
- JobResponse {
- domain: "letsencrypt.org".to_string().into(),
- url_responses: letsencrypt_url_responses,
- discovered_urls: apple_urls,
- },
- JobResponse {
- domain: "apple.com".to_string().into(),
- url_responses: apple_url_responses,
- discovered_urls: letsencrypt_urls,
- },
- ];
- let mut rng = rand::thread_rng();
+ let mut discovered_urls = Vec::new();
+ for _ in 0..3 {
+ let domain = rand_domain();
+
+ for _ in 0..10 {
+ discovered_urls.push(rand_url(&domain));
+ }
+ }
+
+ responses.push(JobResponse {
+ domain: domain.clone(),
+ url_responses,
+ discovered_urls,
+ });
+ }
+
+ responses
+}
+
+pub fn criterion_benchmark(c: &mut Criterion) {
+ let coordinator = CrawlCoordinator::new("data/crawldb", 100, vec![]).unwrap();
for _ in 0..1_000 {
c.bench_function("Add response to coordinator", |b| {
b.iter(|| {
- let response = responses.choose(&mut rng).unwrap();
- coordinator.add_responses(&[response.clone()]).unwrap();
+ let responses = random_responses(1024);
+ coordinator.add_responses(&responses).unwrap();
})
});
diff --git a/core/examples/crawl_coordinator_memory_stress.rs b/core/examples/crawl_coordinator_memory_stress.rs
index 36c4c40e..6f7143c3 100644
--- a/core/examples/crawl_coordinator_memory_stress.rs
+++ b/core/examples/crawl_coordinator_memory_stress.rs
@@ -1,5 +1,5 @@
use anyhow::Result;
-use indicatif::ProgressIterator;
+use indicatif::{ProgressBar, ProgressStyle};
use rand::Rng;
use stract::crawler::{CrawlCoordinator, Domain, JobResponse, UrlResponse};
use url::Url;
@@ -34,6 +34,37 @@ fn rand_url(domain: &Domain) -> Url {
Url::parse(&format!("https://{}/{}", domain.as_str(), path)).unwrap()
}
+fn random_responses(num: usize) -> Vec<JobResponse> {
+ let mut responses = Vec::with_capacity(num);
+ for _ in 0..num {
+ let domain = rand_domain();
+
+ let url_responses = vec![
+ UrlResponse::Success {
+ url: rand_url(&domain)
+ };
+ 100
+ ];
+
+ let mut discovered_urls = Vec::new();
+ for _ in 0..3 {
+ let domain = rand_domain();
+
+ for _ in 0..10 {
+ discovered_urls.push(rand_url(&domain));
+ }
+ }
+
+ responses.push(JobResponse {
+ domain: domain.clone(),
+ url_responses,
+ discovered_urls,
+ });
+ }
+
+ responses
+}
+
fn main() -> Result<()> {
let test_size = 1_000_000_000;
if std::path::Path::new("data/crawldb").exists() {
@@ -42,34 +73,18 @@ fn main() -> Result<()> {
let coordinator = CrawlCoordinator::new("data/crawldb", test_size, vec![])?;
- for _ in (0..test_size as usize).progress() {
- let mut responses = Vec::new();
- for _ in 0..100 {
- let domain = rand_domain();
-
- let url_responses = vec![
- UrlResponse::Success {
- url: rand_url(&domain)
- };
- 100
- ];
-
- let mut discovered_urls = Vec::new();
- for _ in 0..3 {
- let domain = rand_domain();
-
- for _ in 0..10 {
- discovered_urls.push(rand_url(&domain));
- }
- }
-
- responses.push(JobResponse {
- domain: domain.clone(),
- url_responses,
- discovered_urls,
- });
- }
+ let pb = ProgressBar::new(test_size);
+ pb.set_style(
+ ProgressStyle::default_bar()
+ .template(
+ "{spinner:.green} [{elapsed_precise}] [{wide_bar}] {pos:>7}/{len:7} ({per_sec})",
+ )
+ .progress_chars("#>-"),
+ );
+ for _ in 0..test_size {
+ pb.inc(1);
+ let responses = random_responses(1024);
coordinator.add_responses(&responses)?;
}
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index 467d3131..665074ca 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -16,21 +16,20 @@
use std::{
cmp::Ordering,
- collections::{BTreeMap, BinaryHeap, HashMap, VecDeque},
+ collections::{BinaryHeap, HashMap, HashSet, VecDeque},
hash::Hash,
- num::NonZeroUsize,
path::Path,
- sync::{Arc, Mutex, MutexGuard},
};
use itertools::Itertools;
-use lru::LruCache;
use rand::Rng;
use rayon::prelude::IntoParallelRefIterator;
use rayon::prelude::*;
use serde::{Deserialize, Serialize};
use url::Url;
+use crate::intmap::{self, IntMap};
+
use super::{Domain, Job, JobResponse, Result, UrlResponse};
#[derive(Clone, PartialEq, Eq, Serialize, Deserialize)]
@@ -41,7 +40,7 @@ pub enum UrlStatus {
Done,
}
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
pub enum DomainStatus {
Pending,
CrawlInProgress,
@@ -67,8 +66,7 @@ impl AsId for Url {
struct IdTable<T> {
db: rocksdb::DB,
- cache: Mutex<LruCache<Id, T>>,
- cache_size: usize,
+ _marker: std::marker::PhantomData<T>,
}
impl<T> IdTable<T>
@@ -92,112 +90,64 @@ where
options.set_allow_mmap_reads(true);
options.set_allow_mmap_writes(true);
options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);
+ options.optimize_for_point_lookup(512);
let db = rocksdb::DB::open(&options, path.as_ref())?;
- let cache_size = 100_000;
Ok(Self {
db,
- cache: Mutex::new(LruCache::new(NonZeroUsize::new(cache_size + 1).unwrap())),
- cache_size,
+ _marker: std::marker::PhantomData,
})
}
pub fn bulk_insert_ids(&self, items: &[(T, Id)]) -> Result<()> {
- let mut cache = self.cache.lock().unwrap();
+ let mut batch = rocksdb::WriteBatch::default();
for (item, id) in items {
- cache.put(*id, item.clone());
- self.maybe_prune_cache(&mut cache)?;
+ batch.put(bincode::serialize(id)?, bincode::serialize(item)?);
}
- Ok(())
- }
-
- fn maybe_prune_cache(&self, guard: &mut MutexGuard<LruCache<Id, T>>) -> Result<()> {
- // if cache is full, write half of it to disk
- if guard.len() >= self.cache_size {
- let mut batch = rocksdb::WriteBatch::default();
-
- for _ in 0..self.cache_size / 2 {
- if let Some((id, value)) = guard.pop_lru() {
- batch.put(bincode::serialize(&id)?, bincode::serialize(&value)?);
- } else {
- break;
- }
- }
-
- let mut write_options = rocksdb::WriteOptions::default();
- write_options.disable_wal(true);
+ let mut write_options = rocksdb::WriteOptions::default();
+ write_options.disable_wal(true);
- self.db.write_opt(batch, &write_options)?;
- }
+ self.db.write_opt(batch, &write_options)?;
Ok(())
}
- pub fn id(&self, item: &T) -> Result<Id> {
- let mut cache = self.cache.lock().unwrap();
- let id = item.as_id();
-
- // check cache
- if cache.get(&id).is_some() {
- Ok(id)
- } else {
- cache.put(id, item.clone());
- self.maybe_prune_cache(&mut cache)?;
- Ok(id)
- }
- }
-
pub fn value(&self, id: Id) -> Result<Option<T>> {
- let mut cache = self.cache.lock().unwrap();
-
- // check cache
- if let Some(value) = cache.get(&id) {
- return Ok(Some(value.clone()));
- }
-
let id_bytes = bincode::serialize(&id)?;
// check db
let value = self.db.get(id_bytes)?;
- let value: Option<T> = if let Some(value) = &value {
- Some(bincode::deserialize(value)?)
- } else {
- None
- };
-
- if let Some(value) = &value {
- cache.put(id, value.clone());
- self.maybe_prune_cache(&mut cache)?;
+ match value {
+ Some(value) => Ok(Some(bincode::deserialize(&value)?)),
+ None => Ok(None),
}
-
- Ok(value)
}
}
-struct SampledItem<'a, T> {
- item: &'a T,
+struct SampledItem<T> {
+ item: T,
priority: f64,
}
-impl<'a, T> PartialEq for SampledItem<'a, T> {
+impl<T> PartialEq for SampledItem<T> {
fn eq(&self, other: &Self) -> bool {
self.priority == other.priority
}
}
-impl<'a, T> Eq for SampledItem<'a, T> {}
+impl<T> Eq for SampledItem<T> {}
-impl<'a, T> PartialOrd for SampledItem<'a, T> {
+impl<T> PartialOrd for SampledItem<T> {
fn partial_cmp(&self, other: &Self) -> Option<std::cmp::Ordering> {
self.priority.partial_cmp(&other.priority)
}
}
-impl<'a, T> Ord for SampledItem<'a, T> {
+impl<T> Ord for SampledItem<T> {
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
self.priority
.partial_cmp(&other.priority)
@@ -205,10 +155,7 @@ impl<'a, T> Ord for SampledItem<'a, T> {
}
}
-fn weighted_sample<'a, T: 'a>(
- items: impl Iterator<Item = (&'a T, f64)>,
- num_items: usize,
-) -> Vec<&'a T> {
+fn weighted_sample<T>(items: impl Iterator<Item = (T, f64)>, num_items: usize) -> Vec<T> {
let mut sampled_items: BinaryHeap<SampledItem<T>> = BinaryHeap::with_capacity(num_items);
let mut rng = rand::thread_rng();
@@ -235,6 +182,7 @@ struct UrlState {
weight: f64,
status: UrlStatus,
}
+#[derive(Clone, Serialize, Deserialize)]
struct DomainState {
weight: f64,
status: DomainStatus,
@@ -308,18 +256,16 @@ struct UrlToInsert {
}
struct PointDb<K, V> {
- cache: LruCache<K, Arc<Mutex<V>>>,
- cache_size: usize,
db: rocksdb::DB,
- _marker: std::marker::PhantomData<K>,
+ _marker: std::marker::PhantomData<(K, V)>,
}
impl<K, V> PointDb<K, V>
where
- K: serde::Serialize + serde::de::DeserializeOwned + Hash + Eq + Clone,
- V: serde::Serialize + serde::de::DeserializeOwned + Clone + Default + Send + Sync,
+ K: serde::Serialize + serde::de::DeserializeOwned,
+ V: serde::Serialize + serde::de::DeserializeOwned,
{
- pub fn open<P: AsRef<Path>>(path: P, cache_size: usize) -> Result<Self> {
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
let mut options = rocksdb::Options::default();
options.create_if_missing(true);
@@ -334,69 +280,118 @@ where
options.set_allow_mmap_reads(true);
options.set_allow_mmap_writes(true);
options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);
+ options.optimize_for_point_lookup(512);
let db = rocksdb::DB::open(&options, path.as_ref())?;
Ok(Self {
- cache: LruCache::new(NonZeroUsize::new(cache_size + 1).unwrap()),
- cache_size,
db,
_marker: std::marker::PhantomData,
})
}
- pub fn get(&mut self, key: K) -> Result<Arc<Mutex<V>>> {
- // check cache
- if let Some(value) = self.cache.get(&key) {
- return Ok(value.clone());
+ pub fn get(&mut self, key: K) -> Result<Option<V>> {
+ let key_bytes = bincode::serialize(&key)?;
+ let value_bytes = self.db.get(key_bytes)?;
+
+ match value_bytes {
+ Some(value_bytes) => Ok(Some(bincode::deserialize(&value_bytes)?)),
+ None => Ok(None),
}
+ }
- // check db
- let domain_id_bytes = bincode::serialize(&key)?;
+ pub fn put(&mut self, key: K, value: V) -> Result<()> {
+ let key_bytes = bincode::serialize(&key)?;
+ let value_bytes = bincode::serialize(&value)?;
+
+ let mut write_options = rocksdb::WriteOptions::default();
+ write_options.disable_wal(true);
+ self.db.put_opt(key_bytes, value_bytes, &write_options)?;
+
+ Ok(())
+ }
+}
+
+struct DomainStateDb {
+ db: rocksdb::DB,
+}
+
+impl DomainStateDb {
+ pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ let mut options = rocksdb::Options::default();
+
+ options.create_if_missing(true);
+
+ let mut block_options = rocksdb::BlockBasedOptions::default();
+ block_options.set_ribbon_filter(10.0);
+
+ options.set_block_based_table_factory(&block_options);
+ options.set_optimize_filters_for_hits(true);
+ options.set_max_background_jobs(8);
+ options.set_write_buffer_size(512 * 1024 * 1024);
+ options.set_allow_mmap_reads(true);
+ options.set_allow_mmap_writes(true);
+ options.set_compaction_style(rocksdb::DBCompactionStyle::Universal);
+
+ let db = rocksdb::DB::open(&options, path.as_ref())?;
+
+ Ok(Self { db })
+ }
+
+ fn get(&self, domain_id: DomainId) -> Result<Option<DomainState>> {
+ let domain_id_bytes = bincode::serialize(&domain_id)?;
let value_bytes = self.db.get(domain_id_bytes)?;
if let Some(value_bytes) = &value_bytes {
- let value: Arc<Mutex<V>> = bincode::deserialize(value_bytes)?;
- self.cache.put(key, value.clone());
+ let value: DomainState = bincode::deserialize(value_bytes)?;
+ return Ok(Some(value));
+ }
- self.maybe_prune_cache()?;
+ Ok(None)
+ }
- return Ok(value);
- }
+ fn put(&self, domain_id: DomainId, state: DomainState) -> Result<()> {
+ let domain_id_bytes = bincode::serialize(&domain_id)?;
+ let state_bytes = bincode::serialize(&state)?;
+
+ let mut write_options = rocksdb::WriteOptions::default();
+ write_options.disable_wal(true);
+ self.db
+ .put_opt(domain_id_bytes, state_bytes, &write_options)?;
- Ok(Arc::new(Mutex::new(V::default())))
+ Ok(())
}
- fn maybe_prune_cache(&mut self) -> Result<()> {
- // if cache is full, write half of it to disk
- if self.cache.len() >= self.cache_size {
- let mut batch = rocksdb::WriteBatch::default();
+ fn iter(&self) -> impl Iterator<Item = (DomainId, DomainState)> + '_ {
+ let iter = self.db.iterator(rocksdb::IteratorMode::Start);
- for _ in 0..self.cache_size / 2 {
- if let Some((key, value)) = self.cache.pop_lru() {
- let domain_id_bytes = bincode::serialize(&key)?;
- let value_bytes = bincode::serialize(&value)?;
+ iter.filter_map(|r| {
+ let (key, value) = r.ok()?;
+ let domain_id: DomainId = bincode::deserialize(&key).unwrap();
+ let state: DomainState = bincode::deserialize(&value).unwrap();
- batch.put(domain_id_bytes, value_bytes);
- } else {
- break;
- }
- }
+ Some((domain_id, state))
+ })
+ }
+}
- let mut write_options = rocksdb::WriteOptions::default();
- write_options.disable_wal(true);
+impl intmap::Key for UrlId {
+ const BIG_PRIME: Self = Self(335579573203413586826293107669396558523);
- self.db.write_opt(batch, &write_options)?;
- }
+ fn wrapping_mul(self, rhs: Self) -> Self {
+ Self(self.0.wrapping_mul(rhs.0))
+ }
- Ok(())
+ fn bit_and(self, rhs: Self) -> Self {
+ Self(self.0 & rhs.0)
}
- pub fn put(&mut self, key: K, value: Arc<Mutex<V>>) -> Result<()> {
- self.cache.put(key, value);
- self.maybe_prune_cache()?;
+ fn from_usize(val: usize) -> Self {
+ Self(val as u128)
+ }
- Ok(())
+ fn as_usize(self) -> usize {
+ self.0 as usize
}
}
@@ -406,41 +401,68 @@ pub struct CrawlDb {
redirects: RedirectDb,
- domain_state: BTreeMap<DomainId, DomainState>,
+ domain_state: DomainStateDb,
- urls: PointDb<DomainId, BTreeMap<UrlId, UrlState>>,
+ urls: PointDb<DomainId, IntMap<UrlId, UrlState>>,
}
impl CrawlDb {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Self> {
+ if Path::new(path.as_ref()).exists() {
+ return Err(anyhow::anyhow!(
+ "crawl db already exists and might be in incorrect state"
+ ));
+ }
+
let url_ids = IdTable::open(path.as_ref().join("urls").join("ids"))?;
- let domain_ids = IdTable::open(path.as_ref().join("domains"))?;
+ let domain_ids = IdTable::open(path.as_ref().join("domains").join("ids"))?;
let redirects = RedirectDb::open(path.as_ref().join("redirects"))?;
Ok(Self {
url_ids,
domain_ids,
redirects,
- domain_state: BTreeMap::new(),
- urls: PointDb::open(path.as_ref().join("urls").join("states"), 10_000)?,
+ domain_state: DomainStateDb::open(path.as_ref().join("domains").join("states"))?,
+ urls: PointDb::open(path.as_ref().join("urls").join("states"))?,
})
}
pub fn insert_seed_urls(&mut self, urls: &[Url]) -> Result<()> {
+ let domain_ids: HashSet<_> = urls
+ .par_iter()
+ .map(Domain::from)
+ .map(|domain| {
+ let id = domain.as_id();
+ (domain, id)
+ })
+ .collect();
+
+ let domain_ids: Vec<_> = domain_ids.into_iter().collect();
+ self.domain_ids.bulk_insert_ids(&domain_ids)?;
+
+ let url_ids: HashSet<_> = urls
+ .par_iter()
+ .map(|url| (url.clone(), url.as_id()))
+ .collect();
+
+ let url_ids: Vec<_> = url_ids.into_iter().collect();
+ self.url_ids.bulk_insert_ids(&url_ids)?;
+
for url in urls {
- let domain_id = self.domain_ids.id(&Domain::from(url))?.into();
- let url_id = self.url_ids.id(url)?.into();
+ let domain_id = Domain::from(url).as_id().into();
+ let url_id = url.as_id().into();
- self.domain_state
- .entry(domain_id)
- .or_insert_with(|| DomainState {
+ self.domain_state.put(
+ domain_id,
+ DomainState {
weight: 0.0,
status: DomainStatus::Pending,
- });
+ },
+ )?;
- let urls = self.urls.get(domain_id)?;
+ let mut urls = self.urls.get(domain_id)?.unwrap_or_default();
- urls.lock().unwrap().insert(
+ urls.insert(
url_id,
UrlState {
weight: 0.0,
@@ -492,44 +514,42 @@ impl CrawlDb {
{
let domain_id = domain_id.into();
- let domain_state = self
- .domain_state
- .entry(domain_id)
- .or_insert_with(|| DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- });
-
- let url_states = self.urls.get(domain_id)?;
-
- {
- let mut url_states = url_states.lock().unwrap();
- for url in urls {
- let id = self.url_ids.id(&url.url);
+ let mut domain_state = match self.domain_state.get(domain_id)? {
+ Some(state) => state,
+ None => {
+ let state = DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ };
+ self.domain_state.put(domain_id, state.clone())?;
- if id.is_err() {
- continue;
- }
+ state
+ }
+ };
- let id = id.unwrap();
+ let mut url_states = self.urls.get(domain_id)?.unwrap_or_default();
- let url_id: UrlId = id.into();
+ for url in urls {
+ let url_id: UrlId = url.url.as_id().into();
- let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {
- weight: 0.0,
- status: UrlStatus::Pending,
- });
+ let mut url_state = url_states.get(&url_id).cloned().unwrap_or(UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ });
- if url.different_domain {
- url_state.weight += 1.0;
- }
+ if url.different_domain {
+ url_state.weight += 1.0;
+ }
- if url_state.weight > domain_state.weight {
- domain_state.weight = url_state.weight;
- }
+ if url_state.weight > domain_state.weight {
+ domain_state.weight = url_state.weight;
}
+
+ url_states.insert(url_id, url_state);
}
+ self.domain_state.put(domain_id, domain_state)?;
+
self.urls.put(domain_id, url_states)?;
}
@@ -598,57 +618,48 @@ impl CrawlDb {
.collect();
self.domain_ids.bulk_insert_ids(&domain_ids)?;
- for (domain_id, responses) in url_responses.into_iter().filter_map(|(domain, responses)| {
- let domain_id: DomainId = self.domain_ids.id(&domain).ok()?.into();
- Some((domain_id, responses))
+ for (domain_id, responses) in url_responses.into_iter().map(|(domain, responses)| {
+ let domain_id = domain.as_id().into();
+ (domain_id, responses)
}) {
- self.domain_state
- .entry(domain_id)
- .or_insert_with(|| DomainState {
- weight: 0.0,
- status: DomainStatus::Pending,
- });
+ if self.domain_state.get(domain_id)?.is_none() {
+ self.domain_state.put(
+ domain_id,
+ DomainState {
+ weight: 0.0,
+ status: DomainStatus::Pending,
+ },
+ )?;
+ }
+
+ let mut url_states = self.urls.get(domain_id)?.unwrap_or_default();
+ for response in responses {
+ match response {
+ UrlResponse::Success { url } => {
+ let url_id: UrlId = url.as_id().into();
+
+ let mut url_state = url_states.get(&url_id).cloned().unwrap_or(UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ });
+
+ url_state.status = UrlStatus::Done;
- let url_states = self.urls.get(domain_id)?;
- {
- let mut url_states = url_states.lock().unwrap();
- for response in responses {
- match response {
- UrlResponse::Success { url } => {
- let id = self.url_ids.id(&url);
-
- if id.is_err() {
- continue;
- }
-
- let url_id: UrlId = id.unwrap().into();
-
- let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {
- weight: 0.0,
- status: UrlStatus::Pending,
- });
-
- url_state.status = UrlStatus::Done;
- }
- UrlResponse::Failed { url, status_code } => {
- let id = self.url_ids.id(&url);
-
- if id.is_err() {
- continue;
- }
-
- let url_id: UrlId = id.unwrap().into();
-
- let url_state = url_states.entry(url_id).or_insert_with(|| UrlState {
- weight: 0.0,
- status: UrlStatus::Pending,
- });
-
- url_state.status = UrlStatus::Failed { status_code };
- }
- UrlResponse::Redirected { url, new_url } => {
- self.redirects.put(&url, &new_url).ok();
- }
+ url_states.insert(url_id, url_state);
+ }
+ UrlResponse::Failed { url, status_code } => {
+ let url_id: UrlId = url.as_id().into();
+
+ let mut url_state = url_states.get(&url_id).cloned().unwrap_or(UrlState {
+ weight: 0.0,
+ status: UrlStatus::Pending,
+ });
+
+ url_state.status = UrlStatus::Failed { status_code };
+ url_states.insert(url_id, url_state);
+ }
+ UrlResponse::Redirected { url, new_url } => {
+ self.redirects.put(&url, &new_url).ok();
}
}
}
@@ -660,45 +671,38 @@ impl CrawlDb {
}
pub fn set_domain_status(&mut self, domain: &Domain, status: DomainStatus) -> Result<()> {
- let domain_id: DomainId = self.domain_ids.id(domain)?.into();
+ self.domain_ids
+ .bulk_insert_ids(&[(domain.clone(), domain.as_id())])?;
+ let domain_id = domain.id();
- let domain_state = self
- .domain_state
- .entry(domain_id)
- .or_insert_with(|| DomainState {
- weight: 0.0,
- status: status.clone(),
- });
+ let mut domain_state = self.domain_state.get(domain_id)?.unwrap_or(DomainState {
+ weight: 0.0,
+ status,
+ });
domain_state.status = status;
+ self.domain_state.put(domain_id, domain_state)?;
+
Ok(())
}
pub fn sample_domains(&mut self, num_jobs: usize) -> Result<Vec<DomainId>> {
- let available_domains: Vec<_> = self
- .domain_state
- .iter()
- .filter_map(|(id, state)| {
+ let sampled = weighted_sample(
+ self.domain_state.iter().filter_map(|(id, state)| {
if state.status == DomainStatus::Pending {
- Some((*id, state.weight))
+ Some((id, state.weight))
} else {
None
}
- })
- .collect();
-
- let sampled = weighted_sample(
- available_domains.iter().map(|(id, weight)| (id, *weight)),
+ }),
num_jobs,
- )
- .into_iter()
- .copied()
- .collect();
+ );
- for id in &sampled {
- let state = self.domain_state.get_mut(id).unwrap();
+ for id in sampled.iter() {
+ let mut state = self.domain_state.get(*id)?.unwrap();
state.status = DomainStatus::CrawlInProgress;
+ self.domain_state.put(*id, state)?;
}
Ok(sampled)
@@ -707,58 +711,56 @@ impl CrawlDb {
pub fn prepare_jobs(&mut self, domains: &[DomainId], urls_per_job: usize) -> Result<Vec<Job>> {
let mut jobs = Vec::with_capacity(domains.len());
for domain_id in domains {
- let urls = self.urls.get(*domain_id)?;
-
- {
- let mut urls = urls.lock().unwrap();
-
- let sampled: Vec<_> = weighted_sample(
- urls.iter().filter_map(|(id, state)| {
- if state.status == UrlStatus::Pending {
- Some((id, state.weight))
- } else {
- None
- }
- }),
- urls_per_job,
- )
- .into_iter()
- .copied()
- .collect();
-
- for id in &sampled {
- let state = urls.get_mut(id).unwrap();
- state.status = UrlStatus::Crawling;
- }
+ let mut urls = self.urls.get(*domain_id)?.unwrap_or_default();
+
+ let sampled: Vec<_> = weighted_sample(
+ urls.iter().filter_map(|(id, state)| {
+ if state.status == UrlStatus::Pending {
+ Some((id, state.weight))
+ } else {
+ None
+ }
+ }),
+ urls_per_job,
+ )
+ .into_iter()
+ .copied()
+ .collect();
- let domain_state = self.domain_state.get_mut(domain_id).unwrap();
+ for id in &sampled {
+ let state = urls.get_mut(id).unwrap();
+ state.status = UrlStatus::Crawling;
+ }
- domain_state.weight = urls
- .iter()
- .filter_map(|(_, state)| {
- if state.status == UrlStatus::Pending {
- Some(state.weight)
- } else {
- None
- }
- })
- .max_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal))
- .unwrap_or(0.0);
-
- let mut job = Job {
- domain: self.domain_ids.value(domain_id.0)?.unwrap(),
- fetch_sitemap: false, // todo: fetch for new sites
- urls: VecDeque::with_capacity(urls_per_job),
- };
-
- for url_id in sampled {
- let url = self.url_ids.value(url_id.0)?.unwrap();
- job.urls.push_back(url);
- }
+ let mut domain_state = self.domain_state.get(*domain_id)?.unwrap();
+
+ domain_state.weight = urls
+ .iter()
+ .filter_map(|(_, state)| {
+ if state.status == UrlStatus::Pending {
+ Some(state.weight)
+ } else {
+ None
+ }
+ })
+ .max_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal))
+ .unwrap_or(0.0);
- jobs.push(job);
+ self.domain_state.put(*domain_id, domain_state.clone())?;
+
+ let mut job = Job {
+ domain: self.domain_ids.value(domain_id.0)?.unwrap(),
+ fetch_sitemap: false, // todo: fetch for new sites
+ urls: VecDeque::with_capacity(urls_per_job),
+ };
+
+ for url_id in sampled {
+ let url = self.url_ids.value(url_id.0)?.unwrap();
+ job.urls.push_back(url);
}
+ jobs.push(job);
+
self.urls.put(*domain_id, urls)?;
}
@@ -800,13 +802,14 @@ mod tests {
.unwrap();
let domain = Domain::from(&Url::parse("https://example.com").unwrap());
- let domain_id = db.domain_ids.id(&domain).unwrap().into();
+ let domain_id = domain.id();
+
let sample = db.sample_domains(128).unwrap();
assert_eq!(sample.len(), 1);
assert_eq!(&sample[0], &domain_id);
assert_eq!(
- db.domain_state.get(&domain_id).unwrap().status,
+ db.domain_state.get(domain_id).unwrap().unwrap().status,
DomainStatus::CrawlInProgress
);
diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rs
index 6601a77e..1b164aa8 100644
--- a/core/src/crawler/robots_txt.rs
+++ b/core/src/crawler/robots_txt.rs
@@ -145,7 +145,7 @@ impl RobotsTxt {
}
fn update(&mut self, body: String) {
- self.matcher.parse(&body);
+ self.matcher.parse(&body.to_lowercase());
self.sitemap = body
.to_ascii_lowercase()
@@ -164,9 +164,26 @@ mod tests {
#[test]
fn simple() {
- let ua_token = "StractSearch";
+ let ua_token = "StractBot";
let mut robots_txt = RobotsTxt::new(
- r#"User-agent: StractSearch
+ r#"User-agent: StractBot
+ Disallow: /test"#
+ .to_string(),
+ );
+
+ assert!(!robots_txt
+ .matcher
+ .one_agent_allowed_by_robots(ua_token, "http://example.com/test"));
+ assert!(robots_txt
+ .matcher
+ .one_agent_allowed_by_robots(ua_token, "http://example.com/example"));
+ }
+
+ #[test]
+ fn lowercase() {
+ let ua_token = "StractBot";
+ let mut robots_txt = RobotsTxt::new(
+ r#"User-agent: stractbot
Disallow: /test"#
.to_string(),
);
diff --git a/frontend/src/pages/crawler.mdx b/frontend/src/pages/crawler.mdx
index 1aade6ab..6746d3a3 100644
--- a/frontend/src/pages/crawler.mdx
+++ b/frontend/src/pages/crawler.mdx
@@ -4,9 +4,9 @@ title: Stract Crawler
---
# Stract Crawler
-StractBot is the name of our crawler that collects pages from the web in order to index them into the search engine.
+Stract is an [open source](https://github.com/StractOrg/stract/) web search engine. StractBot is the name of our crawler that collects pages from the web in order to build the index.
It is written in Rust and the source code can be found [here](https://github.com/StractOrg/stract/tree/main/core/src/crawler).
-The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; open source search engine; +https://trystract.com/webmasters)`.
+The crawler uses the user agent `Mozilla/5.0 (compatible; StractBot/0.1; +https://trystract.com/webmasters)`.
## Politeness
StractBot is a polite crawler. It respects the [robots.txt](https://en.wikipedia.org/wiki/Robots.txt) file of the website it is crawling and tries to not overload the server.