Skip to content

web search done right

Spec compliant url parser. This is a giant re-write to use the spec compliant 'url' crate. The parser we had before did not follow relative urls correctly, which caused many 404's when crawling. There was probably also a bunch of other issues with the simple parser, so this should make webgraph and everything related to it more robust as well.

Mikkel Denkere28bbd75bfe5
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index cb2de05a..b7a1939f 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -5,9 +5,11 @@ bug reports, feature request, and evangelizing are as valuable.
# Submitting a PR
Check if your issue is already listed [github](https://github.com/StractOrg/stract/issues).
-If it is not, create your own issue.
+If it is not, create your own issue. Please make sure that an issue exists before
+submitting a PR. This will allow us to discuss the issue and make sure that the
+PR is not a waste of your time.
-Please add the following phrase at the end of your commit. `Closes #<Issue Number>`.
+Please add the following phrase at the end of your commit `Closes #<Issue Number>`.
It will automatically link your PR in the issue page. Also, once your PR is merged, it will
closes the issue. If your PR only partially addresses the issue and you would like to
keep it open, just write `See #<Issue Number>`.
diff --git a/Cargo.lock b/Cargo.lock
index 786d041c..0004dcd5 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2,6 +2,15 @@
# It is not intended for manual editing.
version = 3
+[[package]]
+name = "addr2line"
+version = "0.19.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a76fd60b23679b7d19bd066031410fb7e458ccc5e958eb5c325888ce4baedc97"
+dependencies = [
+ "gimli",
+]
+
[[package]]
name = "adler"
version = "1.0.2"
@@ -155,9 +164,12 @@ dependencies = [
[[package]]
name = "anyhow"
-version = "1.0.70"
+version = "1.0.72"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7de8ce5e0f9f8d88245311066a578d72b7af3e7088f32783804676302df237e4"
+checksum = "3b13c32d80ecc7ab747b80c3784bce54ee8a7a0cc4fbda9bf4cda2cf6fe90854"
+dependencies = [
+ "backtrace",
+]
[[package]]
name = "arc-swap"
@@ -416,6 +428,21 @@ dependencies = [
"syn 2.0.15",
]
+[[package]]
+name = "backtrace"
+version = "0.3.67"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "233d376d6d185f2a3093e58f283f60f880315b6c60075b01f36b3b85154564ca"
+dependencies = [
+ "addr2line",
+ "cc",
+ "cfg-if",
+ "libc",
+ "miniz_oxide 0.6.2",
+ "object",
+ "rustc-demangle",
+]
+
[[package]]
name = "base64"
version = "0.13.1"
@@ -1505,9 +1532,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b"
[[package]]
name = "form_urlencoded"
-version = "1.1.0"
+version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a9c384f161156f5260c24a097c56119f9be8c798586aecc13afbcbe7b7e26bf8"
+checksum = "a62bc1cf6f830c2ec14a513a9fb124d0a213a629668a4186f329db21fe045652"
dependencies = [
"percent-encoding",
]
@@ -1718,6 +1745,12 @@ dependencies = [
"weezl",
]
+[[package]]
+name = "gimli"
+version = "0.27.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b6c80984affa11d98d1b88b66ac8853f143217b399d3c74116778ff8fdb4ed2e"
+
[[package]]
name = "glob"
version = "0.3.1"
@@ -1981,9 +2014,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
[[package]]
name = "idna"
-version = "0.3.0"
+version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e14ddfc70884202db2244c223200c204c2bda1bc6e0998d11b5e024d657209e6"
+checksum = "7d20d6b07bfbc108882d88ed8e37d39636dcc260e15e30c45e6ba089610b917c"
dependencies = [
"unicode-bidi",
"unicode-normalization",
@@ -2770,6 +2803,15 @@ version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
+[[package]]
+name = "object"
+version = "0.30.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "03b4680b86d9cfafba8fc491dc9b6df26b68cf40e9e6cd73909194759a63c385"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "once_cell"
version = "1.17.1"
@@ -2973,9 +3015,9 @@ checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
[[package]]
name = "percent-encoding"
-version = "2.2.0"
+version = "2.3.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "478c572c3d73181ff3c2539045f6eb99e5491218eae919370993b890cdbdd98e"
+checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94"
[[package]]
name = "petgraph"
@@ -3237,12 +3279,6 @@ dependencies = [
"unicode-ident",
]
-[[package]]
-name = "psl-types"
-version = "2.0.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33cb294fe86a74cbcf50d4445b37da762029549ebeea341421c7c70370f86cac"
-
[[package]]
name = "ptr_meta"
version = "0.1.4"
@@ -3263,17 +3299,6 @@ dependencies = [
"syn 1.0.109",
]
-[[package]]
-name = "publicsuffix"
-version = "2.2.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96a8c1bda5ae1af7f99a2962e49df150414a43d62404644d98dd5c3a93d07457"
-dependencies = [
- "hashbrown 0.12.3",
- "idna",
- "psl-types",
-]
-
[[package]]
name = "qoi"
version = "0.4.1"
@@ -3657,6 +3682,12 @@ dependencies = [
"serde_derive",
]
+[[package]]
+name = "rustc-demangle"
+version = "0.1.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76"
+
[[package]]
name = "rustc-hash"
version = "1.1.0"
@@ -4142,7 +4173,6 @@ dependencies = [
"once_cell",
"optics",
"parse_wiki_text",
- "publicsuffix",
"quick-xml 0.23.1",
"rand 0.8.5",
"rayon",
@@ -4170,6 +4200,7 @@ dependencies = [
"tower-http",
"tracing",
"tracing-subscriber",
+ "url",
"urlencoding",
"uuid",
"whatlang",
@@ -4916,13 +4947,14 @@ dependencies = [
[[package]]
name = "url"
-version = "2.3.1"
+version = "2.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d68c799ae75762b8c3fe375feb6600ef5602c883c5d21eb51c09f22b83c4643"
+checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb"
dependencies = [
"form_urlencoded",
"idna",
"percent-encoding",
+ "serde",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index c1699707..f0da6f28 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -39,7 +39,6 @@ askama = "0.11.1"
axum = "0.6.9"
axum-extra = { version = "0.6.0", features = ["spa"] }
axum-macros = "0.3.4"
-anyhow = "1.0.58"
html-escape = "0.2.11"
logos = "0.12.1"
csv = "1.1.6"
@@ -86,9 +85,11 @@ torch-sys = "0.13.0"
libc = "0.2.142"
lz-str = "0.2.1"
safetensors = "0.3.1"
-publicsuffix = { version = "2", features = ["hashbrown"] }
robotstxt-with-cache = "0.4.0"
rust-s3 = { version = "0.33.0", features = ["blocking", "tokio"] }
rusqlite = { version = "0.29.0", features = ["bundled", "modern-full", "buildtime_bindgen"] }
async-channel = "1.8.0"
lru = "0.10.0"
+url = { version = "2.4.0", features = ["serde"] }
+anyhow = { version = "1.0.72", features = ["backtrace"] }
+
diff --git a/README.md b/README.md
index b3c5fece..3eedf5b5 100644
--- a/README.md
+++ b/README.md
@@ -38,4 +38,4 @@ You can contact us at [[email protected]](mailto:[email protected]) or in ou
# 🏆 Thank you!
We truly stand on the shoulders of giants and this project would not have been even remotely feasible without them. An especially huge thank you to
* The authors and contributors of Tantivy for providing the inverted index library on which Stract is built.
-* The commoncrawl organization for crawling the web and making the dataset readily available.
\ No newline at end of file
+* The commoncrawl organization for crawling the web and making the dataset readily available. Even though we have our own crawler now, commoncrawl has been a huge help in the early stages of development.
\ No newline at end of file
diff --git a/core/Cargo.toml b/core/Cargo.toml
index a092d90b..417d7763 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -35,7 +35,6 @@ whatlang = { workspace = true }
tokio = { workspace = true }
askama = { workspace = true }
axum = { workspace = true }
-anyhow = { workspace = true }
axum-extra = { workspace = true }
html-escape = { workspace = true }
logos = { workspace = true }
@@ -87,11 +86,12 @@ rand = { workspace = true }
half = { workspace = true }
aes-gcm = { workspace = true }
reqwest-eventsource = { workspace = true }
-publicsuffix = { workspace = true }
robotstxt-with-cache = { workspace = true }
rust-s3 = { workspace = true }
async-channel = { workspace = true }
lru = { workspace = true }
+url = { workspace = true }
+anyhow = { workspace = true }
[build-dependencies]
lalrpop = { workspace = true }
diff --git a/core/benches/crawl-coordinator.rs b/core/benches/crawl-coordinator.rs
index f2755d74..ae2960ff 100644
--- a/core/benches/crawl-coordinator.rs
+++ b/core/benches/crawl-coordinator.rs
@@ -1,1746 +1,1744 @@
// rust-analyzer: disable
use criterion::{criterion_group, criterion_main, Criterion};
use rand::seq::SliceRandom;
-use stract::{
- crawler::{CrawlCoordinator, JobResponse, UrlResponse},
- webpage::Url,
-};
+use stract::crawler::{CrawlCoordinator, JobResponse, UrlResponse};
+use url::Url;
pub fn criterion_benchmark(c: &mut Criterion) {
let coordinator = CrawlCoordinator::new("data/crawldb", 100, vec![]).unwrap();
let letsencrypt_urls = vec![
-Url::from("https://letsencrypt.org/2019/11/20/how-le-runs-ct-logs.html".to_string()),
-Url::from("https://community.letsencrypt.org/t/invalid-certificate-on-mobile-works-on-desktop/24295".to_string()),
-Url::from("https://community.letsencrypt.org/t/frequently-asked-questions-faq".to_string()),
-Url::from("https://community.letsencrypt.org/t/upcoming-intermediate-changes/13106".to_string()),
-Url::from("https://community.letsencrypt.org/t/san-domain-name-mismatch-android-2-windows-xp/4060".to_string()),
-Url::from("https://community.letsencrypt.org/t/small-change-to-end-entity-certificates-cps-url-and-oid-will-not-be-included-from-june-15/198206".to_string()),
-Url::from("https://community.letsencrypt.org/t/web-hosting-who-support-lets-encrypt/6920".to_string()),
-Url::from("https://letsencrypt.org/2022/04/13/receiving-the-levchin-prize.html".to_string()),
-Url::from("https://letsencrypt.org/2020/02/27/one-billion-certs.html".to_string()),
-Url::from("https://letsencrypt.org/docs/integration-guide".to_string()),
-Url::from("https://letsencrypt.org/fr/docs".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-v3.4".to_string()),
-Url::from("https://letsencrypt.org/sr/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/id/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/de/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/he/about".to_string()),
-Url::from("https://letsencrypt.org/da/getting-started".to_string()),
-Url::from("https://letsencrypt.org/es/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/da/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/privacy".to_string()),
-Url::from("https://letsencrypt.org/blog/#main-content".to_string()),
-Url::from("https://letsencrypt.org/pt-br/sponsors".to_string()),
-Url::from("https://letsencrypt.org/documents/LE-SA-v1.0.1-v1.1.1-diff.docx".to_string()),
-Url::from("https://letsencrypt.org/ja/#".to_string()),
-Url::from("https://letsencrypt.org/pt-br/privacy".to_string()),
-Url::from("https://letsencrypt.org/2015/02/19/draft-isrg-cp.html".to_string()),
-Url::from("https://letsencrypt.org/2021/10/21/celebrating-encryption-globally.html".to_string()),
-Url::from("https://letsencrypt.org/docs/lencr.org".to_string()),
-Url::from("https://letsencrypt.org/it/repository".to_string()),
-Url::from("https://community.letsencrypt.org/guidelines".to_string()),
-Url::from("https://letsencrypt.org/vi/donate".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/stats/#".to_string()),
-Url::from("https://letsencrypt.org/sponsors/#".to_string()),
-Url::from("https://letsencrypt.org/fi/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/de/docs".to_string()),
-Url::from("https://letsencrypt.org/vi/docs".to_string()),
-Url::from("https://letsencrypt.org/docs/glossary/#def-root".to_string()),
-Url::from("https://community.letsencrypt.org/u/KOKU9408".to_string()),
-Url::from("https://community.letsencrypt.org/t/unauthorized-certificate-issued-for-domain/200845".to_string()),
-Url::from("https://letsencrypt.org/contact/#".to_string()),
-Url::from("https://letsencrypt.org/2017/10/17/acme-support-in-apache-httpd.html".to_string()),
-Url::from("https://letsencrypt.org/fr/donate".to_string()),
-Url::from("https://letsencrypt.org/docs/ipv6-support".to_string()),
-Url::from("https://community.letsencrypt.org/u/frederik.lohse".to_string()),
-Url::from("https://letsencrypt.org/ko/privacy".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-v2.3".to_string()),
-Url::from("https://letsencrypt.org/fi/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/docs/certificates-for-localhost".to_string()),
-Url::from("https://community.letsencrypt.org/u/Oosterkamp".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v3.2".to_string()),
-Url::from("https://letsencrypt.org/si/sponsors".to_string()),
-Url::from("https://community.letsencrypt.org/t/ssl-certificate-stopped-working/200817".to_string()),
-Url::from("https://community.letsencrypt.org/t/dns-dnssec-error-signal/200808".to_string()),
-Url::from("https://letsencrypt.org/da/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/docs/staging-environment".to_string()),
-Url::from("https://letsencrypt.org/docs/acme-protocol-updates".to_string()),
-Url::from("https://letsencrypt.org/si/#main-content".to_string()),
-Url::from("https://letsencrypt.org/pt-br/repository".to_string()),
-Url::from("https://letsencrypt.org/documents/LE-SA-v1.1.1-v1.2-diff.docx".to_string()),
-Url::from("https://letsencrypt.org/de/#".to_string()),
-Url::from("https://letsencrypt.org/it/contact".to_string()),
-Url::from("https://letsencrypt.org/2016/09/20/what-it-costs-to-run-lets-encrypt.html".to_string()),
-Url::from("https://letsencrypt.org/it/privacy".to_string()),
-Url::from("https://letsencrypt.org/pt-br/#main-content".to_string()),
-Url::from("https://letsencrypt.org/docs/integration-guide/#when-to-renew".to_string()),
-Url::from("https://letsencrypt.org/ko/getting-started".to_string()),
-Url::from("https://letsencrypt.org/fr/#".to_string()),
-Url::from("https://letsencrypt.org/ru/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/hu/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/2015/11/12/public-beta-timing.html/#".to_string()),
-Url::from("https://community.letsencrypt.org/u/chandresh".to_string()),
-Url::from("https://community.letsencrypt.org/t/apache2-server-certbot-unable-to-find-corresponding-http-vhost/200854".to_string()),
-Url::from("https://letsencrypt.org/2015/10/08/isoc-sponsors-lets-encrypt.html".to_string()),
-Url::from("https://letsencrypt.org/ja/repository".to_string()),
-Url::from("https://letsencrypt.org/fi/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/sr/#main-content".to_string()),
-Url::from("https://letsencrypt.org/vi/getting-started".to_string()),
-Url::from("https://letsencrypt.org/2021/02/10/200m-certs-24hrs.html".to_string()),
-Url::from("https://letsencrypt.org/hu/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/fr/contact".to_string()),
-Url::from("https://letsencrypt.org/docs/too-many-registrations-for-this-ip".to_string()),
-Url::from("https://letsencrypt.org/he/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/si/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/sr/stats/#".to_string()),
-Url::from("https://community.letsencrypt.org/u/schoen".to_string()),
-Url::from("https://letsencrypt.org/2023/01/12/eng-culture-at-isrg.html/#main-content".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v4.2".to_string()),
-Url::from("https://letsencrypt.org/ru/contact".to_string()),
-Url::from("https://letsencrypt.org/2014/11/18/announcing-lets-encrypt.html".to_string()),
-Url::from("https://letsencrypt.org/id/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/2016/03/09/le-client-new-home.html".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/2015/10/29/phishing-and-malware.html".to_string()),
-Url::from("https://letsencrypt.org/id/docs".to_string()),
-Url::from("https://letsencrypt.org/ko/sponsors".to_string()),
-Url::from("https://community.letsencrypt.org/u/biwek86619".to_string()),
-Url::from("https://letsencrypt.org/docs/certificate-compatibility".to_string()),
-Url::from("https://letsencrypt.org/fi/stats/#".to_string()),
-Url::from("https://letsencrypt.org/he/docs".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-v3.2".to_string()),
-Url::from("https://letsencrypt.org/ko/stats/#".to_string()),
-Url::from("https://community.letsencrypt.org/t/new-certficiate-not-point-to-the-file/200732".to_string()),
-Url::from("https://letsencrypt.org/de/#main-content".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/getting-started".to_string()),
-Url::from("https://letsencrypt.org/ja/donate".to_string()),
-Url::from("https://letsencrypt.org/de/getinvolved".to_string()),
-Url::from("https://community.letsencrypt.org/c/help/ajuda-em-portugues".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/sponsors".to_string()),
-Url::from("https://letsencrypt.org/2018/04/04/sct-encoding.html".to_string()),
-Url::from("https://letsencrypt.org/about/#".to_string()),
-Url::from("https://community.letsencrypt.org/latest?no_definitions=true&page=1".to_string()),
-Url::from("https://letsencrypt.org/hu/sponsors".to_string()),
-Url::from("https://letsencrypt.org/id/#".to_string()),
-Url::from("https://letsencrypt.org/hu/contact".to_string()),
-Url::from("https://letsencrypt.org/fi/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/he/getting-started".to_string()),
-Url::from("https://letsencrypt.org/docs/#".to_string()),
-Url::from("https://letsencrypt.org/ko/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/uk/#".to_string()),
-Url::from("https://community.letsencrypt.org/t/is-it-wise-to-use-lets-encrypt-certificates-for-a-production-endpoint/200836".to_string()),
-Url::from("https://letsencrypt.org/he/#main-content".to_string()),
-Url::from("https://letsencrypt.org/es/#main-content".to_string()),
-Url::from("https://letsencrypt.org/ko/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://community.letsencrypt.org/u/Nummer378".to_string()),
-Url::from("https://letsencrypt.org/fi/getting-started".to_string()),
-Url::from("https://letsencrypt.org/fi/repository".to_string()),
-Url::from("https://letsencrypt.org/2023/03/23/improving-resliiency-and-reliability-with-ari.html/#main-content".to_string()),
-Url::from("https://letsencrypt.org/pt-br/#".to_string()),
-Url::from("https://letsencrypt.org/he/sponsors".to_string()),
-Url::from("https://letsencrypt.org/de/contact".to_string()),
-Url::from("https://letsencrypt.org/si/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/2020/11/06/own-two-feet.html".to_string()),
-Url::from("https://letsencrypt.org/he/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/ja/docs".to_string()),
-Url::from("https://community.letsencrypt.org/categories".to_string()),
-Url::from("https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der".to_string()),
-Url::from("https://letsencrypt.org/sv/getting-started".to_string()),
-Url::from("https://community.letsencrypt.org/u/josh".to_string()),
-Url::from("https://letsencrypt.org/sr/#".to_string()),
-Url::from("https://letsencrypt.org/vi/privacy".to_string()),
-Url::from("https://community.letsencrypt.org/u/0xffcaee".to_string()),
-Url::from("https://letsencrypt.org/da/sponsors".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/repository".to_string()),
-Url::from("https://letsencrypt.org/id/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/he/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/stats/#".to_string()),
-Url::from("https://letsencrypt.org/2016/08/05/le-root-to-be-trusted-by-mozilla.html".to_string()),
-Url::from("https://community.letsencrypt.org/u/jiapei100".to_string()),
-Url::from("https://community.letsencrypt.org/u/jrsyo".to_string()),
-Url::from("https://letsencrypt.org/ko/contact".to_string()),
-Url::from("https://letsencrypt.org/2022/12/05/ed-letter-2022.html".to_string()),
-Url::from("https://letsencrypt.org/docs/rate-limits".to_string()),
-Url::from("https://community.letsencrypt.org/u/greatlakesclub".to_string()),
-Url::from("https://community.letsencrypt.org/t/o-t-1-options-in-access-log/200818".to_string()),
-Url::from("https://letsencrypt.org/ru/stats/#".to_string()),
-Url::from("https://letsencrypt.org/pt-br/docs".to_string()),
-Url::from("https://letsencrypt.org/si/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/docs/ct-logs".to_string()),
-Url::from("https://letsencrypt.org/sv/repository".to_string()),
-Url::from("https://letsencrypt.org/2016/11/01/launching-our-crowdfunding-campaign.html".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v2.7".to_string()),
-Url::from("https://letsencrypt.org/de/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/es/donate".to_string()),
-Url::from("https://letsencrypt.org/da/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/contact".to_string()),
-Url::from("https://letsencrypt.org/it/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/2016/04/01/legal-transparency-report.html".to_string()),
-Url::from("https://letsencrypt.org/it/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/fr/stats/#growth".to_string()),
-Url::from("https://community.letsencrypt.org/t/openssl-client-compatibility-changes-for-let-s-encrypt-certificates/143816".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v2.5".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/getting-started".to_string()),
-Url::from("https://letsencrypt.org/2017/07/06/wildcard-certificates-coming-jan-2018.html".to_string()),
-Url::from("https://letsencrypt.org/fr/about".to_string()),
-Url::from("https://letsencrypt.org/de/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/uk/privacy".to_string()),
-Url::from("https://letsencrypt.org/2022/09/12/remembering-peter-eckersley.html".to_string()),
-Url::from("https://letsencrypt.org/ru/privacy".to_string()),
-Url::from("https://letsencrypt.org/2016/04/12/leaving-beta-new-sponsors.html".to_string()),
-Url::from("https://community.letsencrypt.org/t/some-vn-cctld-sld-arent-recognized-and-causing-ratelimited-error/200596".to_string()),
-Url::from("https://letsencrypt.org/ko/donate".to_string()),
-Url::from("https://letsencrypt.org/docs/duplicate-certificate-limit".to_string()),
-Url::from("https://letsencrypt.org/pt-br/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/2023/01/12/eng-culture-at-isrg.html/#".to_string()),
-Url::from("https://letsencrypt.org/2020/12/21/extending-android-compatibility.html".to_string()),
-Url::from("https://community.letsencrypt.org/u/Rip".to_string()),
-Url::from("https://letsencrypt.org/de/donate".to_string()),
-Url::from("https://community.letsencrypt.org/u/rmhrisk".to_string()),
-Url::from("https://community.letsencrypt.org/c/help/ayuda-en-espanol/22".to_string()),
-Url::from("https://community.letsencrypt.org/t/acme-v2-production-environment-wildcards/55578".to_string()),
-Url::from("https://letsencrypt.org/vi/stats/#".to_string()),
-Url::from("https://community.letsencrypt.org/t/tls-1-2-and-tls-1-3-need-curve25519-and-curve448-ssl-certificates/200775".to_string()),
-Url::from("https://community.letsencrypt.org/t/dns-problem-query-timed-out-looking-up-a/200751".to_string()),
-Url::from("https://community.letsencrypt.org/u/Nekit".to_string()),
-Url::from("https://letsencrypt.org/privacy/#".to_string()),
-Url::from("https://letsencrypt.org/fi/privacy".to_string()),
-Url::from("https://letsencrypt.org/vi/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/ru/donate".to_string()),
-Url::from("https://letsencrypt.org/da/#".to_string()),
-Url::from("https://letsencrypt.org/vi/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v3.1".to_string()),
-Url::from("https://letsencrypt.org/2015/08/13/lets-encrypt-community-support.html".to_string()),
-Url::from("https://letsencrypt.org/vi/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/fr/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/ru/docs/faq".to_string()),
-Url::from("https://community.letsencrypt.org/t/is-there-anything-new-to-generate-lets-encrypt-certificate-with-docker-and-nginx/200837".to_string()),
-Url::from("https://letsencrypt.org/uk/docs".to_string()),
-Url::from("https://letsencrypt.org/si/about".to_string()),
-Url::from("https://community.letsencrypt.org/t/certbot-renew-task-windows-setting/200282".to_string()),
-Url::from("https://community.letsencrypt.org/u/keeptake".to_string()),
-Url::from("https://letsencrypt.org/vi/about".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/docs".to_string()),
-Url::from("https://letsencrypt.org/docs/faq/#main-content".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v2.1".to_string()),
-Url::from("https://community.letsencrypt.org/u/Bruce5051".to_string()),
-Url::from("https://letsencrypt.org/si/privacy".to_string()),
-Url::from("https://letsencrypt.org/ru/getting-started".to_string()),
-Url::from("https://letsencrypt.org/sr/about".to_string()),
-Url::from("https://letsencrypt.org/da/getinvolved".to_string()),
-Url::from("https://community.letsencrypt.org/t/why-i-cant-acess-this-site/200850".to_string()),
-Url::from("https://letsencrypt.org/2015/12/03/facebook-sponsorship.html".to_string()),
-Url::from("https://cp.letsencrypt.org".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/docs".to_string()),
-Url::from("https://letsencrypt.org/id/#main-content".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v2.3".to_string()),
-Url::from("https://letsencrypt.org/fr/repository".to_string()),
-Url::from("https://letsencrypt.org/sv/stats/#".to_string()),
-Url::from("https://letsencrypt.org/2015/11/12/public-beta-timing.html/#main-content".to_string()),
-Url::from("https://letsencrypt.org/pt-br/donate".to_string()),
-Url::from("https://letsencrypt.org/ja/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/sr/docs/faq".to_string()),
-Url::from("https://community.letsencrypt.org/u/jacordar".to_string()),
-Url::from("https://letsencrypt.org/sr/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/es/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/2017/01/06/le-2016-in-review.html".to_string()),
-Url::from("https://letsencrypt.org/2022/02/25/ford-foundation.html".to_string()),
-Url::from("https://letsencrypt.org/2023/03/23/improving-resliiency-and-reliability-with-ari.html/#".to_string()),
-Url::from("https://letsencrypt.org/si/#".to_string()),
-Url::from("https://letsencrypt.org/docs/caa".to_string()),
-Url::from("https://letsencrypt.org/es/docs".to_string()),
-Url::from("https://letsencrypt.org/2016/10/01/legal-transparency-report.html".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v2.6".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/#".to_string()),
-Url::from("https://letsencrypt.org/fi/sponsors".to_string()),
-Url::from("https://letsencrypt.org/hu/stats/#".to_string()),
-Url::from("https://community.letsencrypt.org/u/mlen".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/about".to_string()),
-Url::from("https://letsencrypt.org/sv/about".to_string()),
-Url::from("https://letsencrypt.org/docs/dst-root-ca-x3-expiration-september-2021/#main-content".to_string()),
-Url::from("https://letsencrypt.org/ja/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/sv/#".to_string()),
-Url::from("https://letsencrypt.org/ru/docs".to_string()),
-Url::from("https://letsencrypt.org/docs/allow-port-80".to_string()),
-Url::from("https://letsencrypt.org/2015/06/04/isrg-ca-certs.html".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v4.5".to_string()),
-Url::from("https://letsencrypt.org/es/sponsors".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/2015/12/03/entering-public-beta.html/#".to_string()),
-Url::from("https://letsencrypt.org/2016/10/21/introducing-idn-support.html".to_string()),
-Url::from("https://letsencrypt.org/uk/getting-started".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-v2.6".to_string()),
-Url::from("https://letsencrypt.org/certificates".to_string()),
-Url::from("https://community.letsencrypt.org/t/thunderbird-exim-dovecot-cert-settings/200882".to_string()),
-Url::from("https://letsencrypt.org/fi/donate".to_string()),
-Url::from("https://letsencrypt.org/ru/sponsors".to_string()),
-Url::from("https://community.letsencrypt.org/u/rahvee_tufts".to_string()),
-Url::from("https://letsencrypt.org/ko/docs".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-cps-v5.0".to_string()),
-Url::from("https://letsencrypt.org/ru/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/contact".to_string()),
-Url::from("https://letsencrypt.org/privacy/#main-content".to_string()),
-Url::from("https://community.letsencrypt.org/u/lordspace".to_string()),
-Url::from("https://letsencrypt.org/docs/glossary".to_string()),
-Url::from("https://letsencrypt.org/sr/docs".to_string()),
-Url::from("https://letsencrypt.org/sv/docs".to_string()),
-Url::from("https://letsencrypt.org/2020/09/17/new-root-and-intermediates.html".to_string()),
-Url::from("https://letsencrypt.org/pt-br/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/he/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-v2.5".to_string()),
-Url::from("https://letsencrypt.org/uk/stats/#".to_string()),
-Url::from("https://letsencrypt.org/fr/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/2015/06/16/lets-encrypt-launch-schedule.html".to_string()),
-Url::from("https://letsencrypt.org/uk/docs/faq".to_string()),
-Url::from("https://community.letsencrypt.org/t/welcome-to-lets-encrypt-community-support/8".to_string()),
-Url::from("https://letsencrypt.org/2019/10/09/onboarding-your-customers-with-lets-encrypt-and-acme.html".to_string()),
-Url::from("https://community.letsencrypt.org/t/no-renewal-config-found/200878".to_string()),
-Url::from("https://letsencrypt.org/it/donate".to_string()),
-Url::from("https://letsencrypt.org/he/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/id/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/hu/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v4.1".to_string()),
-Url::from("https://community.letsencrypt.org/t/reminder-email-after-adding-or-removing-names/200846".to_string()),
-Url::from("https://community.letsencrypt.org/t/dh-2048-to-dh-4096/200885".to_string()),
-Url::from("https://letsencrypt.org/hu/#main-content".to_string()),
-Url::from("https://letsencrypt.org/ko/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/docs/godaddy".to_string()),
-Url::from("https://letsencrypt.org/ja/about".to_string()),
-Url::from("https://letsencrypt.org/2021/10/28/tls-simply-and-automatically.html".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v2.0".to_string()),
-Url::from("https://letsencrypt.org/2018/12/31/looking-forward-to-2019.html".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/stats/#growth".to_string()),
-Url::from("https://community.letsencrypt.org/u/kaponir".to_string()),
-Url::from("https://letsencrypt.org/sr/getting-started".to_string()),
-Url::from("https://letsencrypt.org/sr/privacy".to_string()),
-Url::from("https://letsencrypt.org/vi/contact".to_string()),
-Url::from("https://letsencrypt.org/2016/10/27/first-grant-ford-foundation.html".to_string()),
-Url::from("https://letsencrypt.org/sv/stats/#main-content".to_string()),
-Url::from("https://community.letsencrypt.org/t/multiple-ssl-with-a-single-mail-server/200804".to_string()),
-Url::from("https://letsencrypt.org/it/sponsors".to_string()),
-Url::from("https://letsencrypt.org/fi/#".to_string()),
-Url::from("https://letsencrypt.org/da/docs".to_string()),
-Url::from("https://community.letsencrypt.org/c/issuance-tech-questions/12".to_string()),
-Url::from("https://community.letsencrypt.org/t/mastodon-type-unauthorized-invalid-response-from-404/200815".to_string()),
-Url::from("https://letsencrypt.org/docs/challenge-types/#dns-01-challenge".to_string()),
-Url::from("https://cps.letsencrypt.org".to_string()),
-Url::from("https://letsencrypt.org/pt-br/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v3.0".to_string()),
-Url::from("https://letsencrypt.org/he/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/fr/getting-started".to_string()),
-Url::from("https://letsencrypt.org/es/#".to_string()),
-Url::from("https://letsencrypt.org/uk/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/2018/08/06/trusted-by-all-major-root-programs.html".to_string()),
-Url::from("https://letsencrypt.org/id/about".to_string()),
-Url::from("https://letsencrypt.org/id/stats/#".to_string()),
-Url::from("https://letsencrypt.org/sv/privacy".to_string()),
-Url::from("https://letsencrypt.org/2023/01/19/renewing-sponsors.html/#".to_string()),
-Url::from("https://letsencrypt.org/2019/05/15/introducing-oak-ct-log.html".to_string()),
-Url::from("https://letsencrypt.org/ja/stats/#growth".to_string()),
-Url::from("https://community.letsencrypt.org/latest.rss".to_string()),
-Url::from("https://letsencrypt.org/2015/04/09/isrg-lf-collaboration.html".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/fr/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/#main-content".to_string()),
-Url::from("https://letsencrypt.org/ru/#main-content".to_string()),
-Url::from("https://letsencrypt.org/he/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/docs/account-id".to_string()),
-Url::from("https://letsencrypt.org/da/about".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/sponsors".to_string()),
-Url::from("https://letsencrypt.org/da/stats/#".to_string()),
-Url::from("https://letsencrypt.org/de/about".to_string()),
-Url::from("https://letsencrypt.org/docs/challenge-types".to_string()),
-Url::from("https://letsencrypt.org/2015/08/07/updated-lets-encrypt-launch-schedule.html".to_string()),
-Url::from("https://letsencrypt.org/it/getting-started".to_string()),
-Url::from("https://letsencrypt.org/uk/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/hu/donate".to_string()),
-Url::from("https://letsencrypt.org/sr/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/fr/#main-content".to_string()),
-Url::from("https://letsencrypt.org/id/contact".to_string()),
-Url::from("https://letsencrypt.org/de/privacy".to_string()),
-Url::from("https://community.letsencrypt.org/t/help-thread-for-dst-root-ca-x3-expiration-september-2021/149190".to_string()),
-Url::from("https://letsencrypt.org/ru/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/sv/donate".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v2.9".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/repository".to_string()),
-Url::from("https://letsencrypt.org/hu/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/pt-br/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/hu/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/repository/#main-content".to_string()),
-Url::from("https://letsencrypt.org/ko/#".to_string()),
-Url::from("https://letsencrypt.org/2015/10/19/lets-encrypt-is-trusted.html".to_string()),
-Url::from("https://letsencrypt.org/sv/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v4.0".to_string()),
-Url::from("https://letsencrypt.org/he/stats/#".to_string()),
-Url::from("https://community.letsencrypt.org/u/xtronics".to_string()),
-Url::from("https://letsencrypt.org/es/stats/#".to_string()),
-Url::from("https://letsencrypt.org/id/privacy".to_string()),
-Url::from("https://community.letsencrypt.org/u/robouden".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/ko/#main-content".to_string()),
-Url::from("https://community.letsencrypt.org/u/Osiris".to_string()),
-Url::from("https://letsencrypt.org/docs/#main-content".to_string()),
-Url::from("https://community.letsencrypt.org/u/jvanasco".to_string()),
-Url::from("https://letsencrypt.org/2015/12/21/ovh-sponsorship.html".to_string()),
-Url::from("https://letsencrypt.org/si/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/es/contact".to_string()),
-Url::from("https://letsencrypt.org/es/about".to_string()),
-Url::from("https://letsencrypt.org/2016/03/08/our-millionth-cert.html".to_string()),
-Url::from("https://letsencrypt.org/docs/revoking".to_string()),
-Url::from("https://letsencrypt.org/si/repository".to_string()),
-Url::from("https://letsencrypt.org/de/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/vi/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://community.letsencrypt.org/u/mholt".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-v3.0".to_string()),
-Url::from("https://community.letsencrypt.org/t/renewal-not-avaiable/200869".to_string()),
-Url::from("https://letsencrypt.org/de/getting-started".to_string()),
-Url::from("https://letsencrypt.org/2019/04/15/transitioning-to-isrg-root.html".to_string()),
-Url::from("https://letsencrypt.org/2021/09/14/speed-at-scale-shopify.html".to_string()),
-Url::from("https://community.letsencrypt.org/c/client-dev/14".to_string()),
-Url::from("https://letsencrypt.org/getinvolved/#main-content".to_string()),
-Url::from("https://letsencrypt.org/ja/stats/#".to_string()),
-Url::from("https://letsencrypt.org/ru/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/id/getting-started".to_string()),
-Url::from("https://letsencrypt.org/it/#main-content".to_string()),
-Url::from("https://community.letsencrypt.org/u/_az".to_string()),
-Url::from("https://letsencrypt.org/it/docs".to_string()),
-Url::from("https://letsencrypt.org/ko/repository".to_string()),
-Url::from("https://community.letsencrypt.org/u/rg305".to_string()),
-Url::from("https://letsencrypt.org/id/sponsors".to_string()),
-Url::from("https://letsencrypt.org/2022/04/28/database-to-app-tls.html".to_string()),
-Url::from("https://letsencrypt.org/uk/trademarks".to_string()),
-Url::from("https://letsencrypt.org/sr/sponsors".to_string()),
-Url::from("https://letsencrypt.org/fi/about".to_string()),
-Url::from("https://letsencrypt.org/id/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/it/about".to_string()),
-Url::from("https://letsencrypt.org/fi/#main-content".to_string()),
-Url::from("https://community.letsencrypt.org/u/linkp".to_string()),
-Url::from("https://community.letsencrypt.org/tos".to_string()),
-Url::from("https://letsencrypt.org/ja/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/upcoming-features".to_string()),
-Url::from("https://letsencrypt.org/uk/donate".to_string()),
-Url::from("https://community.letsencrypt.org/u/aarongable".to_string()),
-Url::from("https://community.letsencrypt.org/u/Mahmoud1".to_string()),
-Url::from("https://letsencrypt.org/sr/stats/#percent-pageloads".to_string()),
-Url::from("https://community.letsencrypt.org/u/yyny".to_string()),
-Url::from("https://letsencrypt.org/ja/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://community.letsencrypt.org/u/petercooperjr".to_string()),
-Url::from("https://letsencrypt.org/da/donate".to_string()),
-Url::from("https://letsencrypt.org/id/repository".to_string()),
-Url::from("https://letsencrypt.org/si/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/vi/#".to_string()),
-Url::from("https://letsencrypt.org/it/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/es/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/id/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/hu/docs".to_string()),
-Url::from("https://letsencrypt.org/ko/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/donate/#".to_string()),
-Url::from("https://letsencrypt.org/blog/#".to_string()),
-Url::from("https://letsencrypt.org/vi/stats/#growth".to_string()),
-Url::from("https://community.letsencrypt.org/u/cocolinofan".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-v2.0".to_string()),
-Url::from("https://letsencrypt.org/sv/#main-content".to_string()),
-Url::from("https://letsencrypt.org/fr/privacy".to_string()),
-Url::from("https://letsencrypt.org/de/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/ru/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/si/docs".to_string()),
-Url::from("https://letsencrypt.org/getinvolved/#".to_string()),
-Url::from("https://letsencrypt.org/hu/#".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/#main-content".to_string()),
-Url::from("https://letsencrypt.org/2023/05/24/isrg-10th-anniversary.html/#".to_string()),
-Url::from("https://letsencrypt.org/2019/02/12/facebook-expands-support-for-letsencrypt.html".to_string()),
-Url::from("https://letsencrypt.org/hu/repository".to_string()),
-Url::from("https://community.letsencrypt.org/t/production-chain-changes/150739/4".to_string()),
-Url::from("https://letsencrypt.org/docs/dst-root-ca-x3-expiration-september-2021/#".to_string()),
-Url::from("https://letsencrypt.org/ko/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/ja/privacy".to_string()),
-Url::from("https://letsencrypt.org/2021/12/16/ed-letter-2021.html".to_string()),
-Url::from("https://letsencrypt.org/2017/03/23/ovh-platinum-renewal.html".to_string()),
-Url::from("https://community.letsencrypt.org/u/smyasoedov".to_string()),
-Url::from("https://letsencrypt.org/es/privacy".to_string()),
-Url::from("https://letsencrypt.org/2016/06/22/https-progress-june-2016.html".to_string()),
-Url::from("https://letsencrypt.org/2021/01/21/next-gen-database-servers.html".to_string()),
-Url::from("https://letsencrypt.org/da/privacy".to_string()),
-Url::from("https://letsencrypt.org/fr/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/2016/07/26/full-ipv6-support.html".to_string()),
-Url::from("https://letsencrypt.org/he/privacy".to_string()),
-Url::from("https://letsencrypt.org/2022/05/19/nurturing-ct-log-growth.html".to_string()),
-Url::from("https://letsencrypt.org/2015/07/01/legal-transparency-report.html".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-v2.1".to_string()),
-Url::from("https://community.letsencrypt.org/t/certbot-invalid-response-http-www-example-org-well-known-acme-challenge/200841".to_string()),
-Url::from("https://letsencrypt.org/hu/getting-started".to_string()),
-Url::from("https://letsencrypt.org/sr/repository".to_string()),
-Url::from("https://community.letsencrypt.org/t/production-chain-changes/150739".to_string()),
-Url::from("https://letsencrypt.org/docs/faq/#".to_string()),
-Url::from("https://letsencrypt.org/es/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/vi/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/security_letsencrypt.org-publickey.asc".to_string()),
-Url::from("https://community.letsencrypt.org/t/using-staging-and-certbot-saves-stuff-in-live-directory/200780".to_string()),
-Url::from("https://letsencrypt.org/2023/01/19/renewing-sponsors.html/#main-content".to_string()),
-Url::from("https://letsencrypt.org/contact/#main-content".to_string()),
-Url::from("https://letsencrypt.org/2015/04/14/ncc-group-audit.html".to_string()),
-Url::from("https://letsencrypt.org/it/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/ja/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/2017/12/07/looking-forward-to-2018.html".to_string()),
-Url::from("https://letsencrypt.org/pt-br/about".to_string()),
-Url::from("https://letsencrypt.org/sr/contact".to_string()),
-Url::from("https://letsencrypt.org/2015/11/09/why-90-days.html/#".to_string()),
-Url::from("https://letsencrypt.org/pt-br/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/uk/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-v3.3".to_string()),
-Url::from("https://letsencrypt.org/sr/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/si/stats/#".to_string()),
-Url::from("https://letsencrypt.org/2017/06/14/acme-v2-api.html".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/about".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-v3.1".to_string()),
-Url::from("https://community.letsencrypt.org/t/hello-can-i-use-lets-encrypt-for-postfix/200169".to_string()),
-Url::from("https://letsencrypt.org/hu/about".to_string()),
-Url::from("https://letsencrypt.org/getting-started/#".to_string()),
-Url::from("https://letsencrypt.org/2015/12/03/entering-public-beta.html/#main-content".to_string()),
-Url::from("https://letsencrypt.org/sv/contact".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v4.4".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-v2.2".to_string()),
-Url::from("https://letsencrypt.org/docs/failed-validation-limit".to_string()),
-Url::from("https://letsencrypt.org/sr/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/fr/stats/#".to_string()),
-Url::from("https://community.letsencrypt.org/u/system".to_string()),
-Url::from("https://community.letsencrypt.org/c/issuance-policy/11".to_string()),
-Url::from("https://community.letsencrypt.org/u/knX5Fw".to_string()),
-Url::from("https://letsencrypt.org/da/repository".to_string()),
-Url::from("https://community.letsencrypt.org/t/acmeisuptime-com/200022".to_string()),
-Url::from("https://letsencrypt.org/ru/repository".to_string()),
-Url::from("https://letsencrypt.org/2015/05/21/draft-le-sa.html".to_string()),
-Url::from("https://letsencrypt.org/how-it-works".to_string()),
-Url::from("https://letsencrypt.org/vi/#main-content".to_string()),
-Url::from("https://letsencrypt.org/fi/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-v2.4".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/uk/contact".to_string()),
-Url::from("https://letsencrypt.org/fr/sponsors".to_string()),
-Url::from("https://letsencrypt.org/it/#".to_string()),
-Url::from("https://letsencrypt.org/2023/05/24/isrg-10th-anniversary.html/#main-content".to_string()),
-Url::from("https://letsencrypt.org/vi/sponsors".to_string()),
-Url::from("https://letsencrypt.org/2015/09/14/our-first-cert.html".to_string()),
-Url::from("https://letsencrypt.org/es/getting-started".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v2.4".to_string()),
-Url::from("https://letsencrypt.org/ko/about".to_string()),
-Url::from("https://letsencrypt.org/he/contact".to_string()),
-Url::from("https://letsencrypt.org/uk/about".to_string()),
-Url::from("https://letsencrypt.org/it/stats/#main-content".to_string()),
-Url::from("https://community.letsencrypt.org/c/help/aide-en-francais".to_string()),
-Url::from("https://letsencrypt.org/sponsors/#main-content".to_string()),
-Url::from("https://letsencrypt.org/sv/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v3.3".to_string()),
-Url::from("https://letsencrypt.org/he/#".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-cps-v5.1".to_string()),
-Url::from("https://letsencrypt.org/fi/stats/#percent-pageloads".to_string()),
-Url::from("https://community.letsencrypt.org/posts.rss".to_string()),
-Url::from("https://letsencrypt.org/documents/LE-SA-v1.2-v1.3-diff.docx".to_string()),
-Url::from("https://community.letsencrypt.org/u/chit164".to_string()),
-Url::from("https://letsencrypt.org/da/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/uk/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/si/contact".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/stats/#daily-issuance".to_string()),
-Url::from("https://community.letsencrypt.org/t/renew-dry-run-fails-with-unable-to-determine-base-domain-error/200831".to_string()),
-Url::from("https://letsencrypt.org/sv/stats/#percent-pageloads".to_string()),
-Url::from("https://community.letsencrypt.org/u/webprofusion".to_string()),
-Url::from("https://letsencrypt.org/uk/repository".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v2.2".to_string()),
-Url::from("https://letsencrypt.org/uk/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/ja/getting-started".to_string()),
-Url::from("https://letsencrypt.org/2016/10/24/squarespace-ocsp-impl.html".to_string()),
-Url::from("https://letsencrypt.org/de/sponsors".to_string()),
-Url::from("https://letsencrypt.org/2016/06/23/defending-our-brand.html".to_string()),
-Url::from("https://letsencrypt.org/2015/04/23/draft-isrg-cp-cps.html".to_string()),
-Url::from("https://community.letsencrypt.org/u/orangepizza".to_string()),
-Url::from("https://community.letsencrypt.org/t/problema-de-conexion-connection-problem/200673".to_string()),
-Url::from("https://letsencrypt.org/id/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/docs/expiration-emails".to_string()),
-Url::from("https://letsencrypt.org/it/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/uk/#main-content".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/es/repository".to_string()),
-Url::from("https://letsencrypt.org/donate/#main-content".to_string()),
-Url::from("https://letsencrypt.org/de/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/donate".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cp-v2.7".to_string()),
-Url::from("https://community.letsencrypt.org/u/JoseGR02".to_string()),
-Url::from("https://letsencrypt.org/2020/12/28/executive-director-letter.html".to_string()),
-Url::from("https://letsencrypt.org/ja/sponsors".to_string()),
-Url::from("https://letsencrypt.org/pt-br/contact".to_string()),
-Url::from("https://community.letsencrypt.org/t/dns-01-challenge-start-to-failure-since-24-at-22h/200816".to_string()),
-Url::from("https://letsencrypt.org/ru/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/ko/stats/#percent-pageloads".to_string()),
-Url::from("https://community.letsencrypt.org/opensearch.xml".to_string()),
-Url::from("https://letsencrypt.org/pt-br/stats/#".to_string()),
-Url::from("https://letsencrypt.org/fi/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/hu/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/privacy".to_string()),
-Url::from("https://letsencrypt.org/2020/02/19/multi-perspective-validation.html".to_string()),
-Url::from("https://letsencrypt.org/docs/client-options".to_string()),
-Url::from("https://letsencrypt.org/uk/sponsors".to_string()),
-Url::from("https://letsencrypt.org/pt-br/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/getting-started/#main-content".to_string()),
-Url::from("https://letsencrypt.org/repository/#".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v2.8".to_string()),
-Url::from("https://community.letsencrypt.org/t/thoughts-from-starting-to-play-with-ari/200276".to_string()),
-Url::from("https://letsencrypt.org/ru/#".to_string()),
-Url::from("https://community.letsencrypt.org/u/Krischu".to_string()),
-Url::from("https://letsencrypt.org/sr/donate".to_string()),
-Url::from("https://letsencrypt.org/docs/faq/#technical".to_string()),
-Url::from("https://letsencrypt.org/si/stats/#main-content".to_string()),
-Url::from("https://community.letsencrypt.org/u/vrubiella".to_string()),
-Url::from("https://letsencrypt.org/fi/contact".to_string()),
-Url::from("https://letsencrypt.org/pt-br/stats/#main-content".to_string()),
-Url::from("https://community.letsencrypt.org/u/MikeMcQ".to_string()),
-Url::from("https://letsencrypt.org/es/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/he/repository".to_string()),
-Url::from("https://community.letsencrypt.org/u/mcpherrinm".to_string()),
-Url::from("https://letsencrypt.org/vi/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/id/donate".to_string()),
-Url::from("https://letsencrypt.org/da/stats/#daily-issuance".to_string()),
-Url::from("https://community.letsencrypt.org/u/phonegal2".to_string()),
-Url::from("https://letsencrypt.org/sv/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/si/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/2019/03/11/acme-protocol-ietf-standard.html".to_string()),
-Url::from("https://letsencrypt.org/si/getting-started".to_string()),
-Url::from("https://letsencrypt.org/fi/docs".to_string()),
-Url::from("https://letsencrypt.org/2022/12/15/ocspcaching.html/#".to_string()),
-Url::from("https://letsencrypt.org/da/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/he/donate".to_string()),
-Url::from("https://letsencrypt.org/docs/faq/#general".to_string()),
-Url::from("https://letsencrypt.org/fr/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/2015/11/09/why-90-days.html/#main-content".to_string()),
-Url::from("https://letsencrypt.org/fr/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/donate".to_string()),
-Url::from("https://letsencrypt.org/vi/repository".to_string()),
-Url::from("https://letsencrypt.org/ja/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/ko/stats/#main-content".to_string()),
-Url::from("https://community.letsencrypt.org/u/mason8592".to_string()),
-Url::from("https://letsencrypt.org/2022/09/07/new-life-for-crls.html".to_string()),
-Url::from("https://letsencrypt.org/2022/12/15/ocspcaching.html/#main-content".to_string()),
-Url::from("https://letsencrypt.org/ja/#main-content".to_string()),
-Url::from("https://letsencrypt.org/2017/06/28/hundred-million-certs.html".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/uk/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/pt-br/getting-started".to_string()),
-Url::from("https://letsencrypt.org/da/contact".to_string()),
-Url::from("https://letsencrypt.org/hu/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/de/repository".to_string()),
-Url::from("https://letsencrypt.org/it/stats/#".to_string()),
-Url::from("https://letsencrypt.org/es/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/it/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/da/#main-content".to_string()),
-Url::from("https://letsencrypt.org/es/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/hu/privacy".to_string()),
-Url::from("https://letsencrypt.org/si/donate".to_string()),
-Url::from("https://letsencrypt.org/about/#main-content".to_string()),
-Url::from("https://letsencrypt.org/de/stats/#".to_string()),
-Url::from("https://letsencrypt.org/ru/about".to_string()),
-Url::from("https://community.letsencrypt.org/u/letsuser".to_string()),
-Url::from("https://community.letsencrypt.org/u/nigihayami".to_string()),
-Url::from("https://letsencrypt.org/sv/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/sv/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://community.letsencrypt.org/c/help/13".to_string()),
-Url::from("https://letsencrypt.org/sv/sponsors".to_string()),
-Url::from("https://letsencrypt.org/documents/isrg-cps-v4.3".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/#".to_string()),
-Url::from("https://letsencrypt.org/trademarks".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/2021/10/01/cert-chaining-help.html".to_string()),
-Url::from("https://community.letsencrypt.org/t/get-error-ref-5401-then-now-ref-5017/200867".to_string()),
-Url::from("https://letsencrypt.org/ja/contact".to_string()),
-Url::from("https://www.letsencrypt.org".to_string()),
-Url::from("https://letsencrypt.org/ja/docs/allow-port-80".to_string()),
-Url::from("https://letsencrypt.org/docs/rate-limits#:~:text=limit%20of%205%20per%20week".to_string()),
-Url::from("https://letsencrypt.org/docs/faq/#what-ip-addresses-does-let-s-encrypt-use-to-validate-my-web-server".to_string()),
-Url::from("https://community.letsencrypt.org/t/revoking-certain-certificates-on-march-4/114864".to_string()),
-Url::from("https://letsencrypt.org/become-a-sponsor".to_string()),
-Url::from("https://letsencrypt.org/?ref=blog.sean-wright.com".to_string()),
-Url::from("https://letsencrypt.org/?ref=spreadprivacy.com".to_string()),
-Url::from("https://letsencrypt.org/howitworks/technology".to_string()),
-Url::from("https://community.letsencrypt.org/t/beta-program-announcements/1631/6".to_string()),
-Url::from("https://acme-v01.api.letsencrypt.org/directory".to_string()),
-Url::from("https://acme-v01.api.letsencrypt.org/directory%22".to_string()),
-Url::from("https://letsencrypt.org/feed.xml".to_string()),
-Url::from("https://letsencrypt.org/stats".to_string()),
-Url::from("https://letsencrypt.org".to_string()),
-Url::from("https://letsencrypt.org/2015/11/12/public-beta-timing.html".to_string()),
-Url::from("https://letsencrypt.org/2015/12/03/entering-public-beta.html".to_string()),
-Url::from("https://letsencrypt.org/2015/11/09/why-90-days.html".to_string()),
-Url::from("https://letsencrypt.org/2023/03/23/improving-resliiency-and-reliability-with-ari.html".to_string()),
-Url::from("https://letsencrypt.org/vi".to_string()),
-Url::from("https://letsencrypt.org/blog".to_string()),
-Url::from("https://letsencrypt.org/stats/#growth".to_string()),
-Url::from("https://letsencrypt.org/si".to_string()),
-Url::from("https://letsencrypt.org/docs/faq".to_string()),
-Url::from("https://letsencrypt.org/uk/stats".to_string()),
-Url::from("https://letsencrypt.org/contact".to_string()),
-Url::from("https://letsencrypt.org/zh-tw".to_string()),
-Url::from("https://letsencrypt.org/he".to_string()),
-Url::from("https://letsencrypt.org/stats/#main-content".to_string()),
-Url::from("https://letsencrypt.org/fi".to_string()),
-Url::from("https://letsencrypt.org/stats/#daily-issuance".to_string()),
-Url::from("https://letsencrypt.org/vi/stats".to_string()),
-Url::from("https://letsencrypt.org/fr/stats".to_string()),
-Url::from("https://letsencrypt.org/pt-br".to_string()),
-Url::from("https://letsencrypt.org/stats/#".to_string()),
-Url::from("https://letsencrypt.org/#main-content".to_string()),
-Url::from("https://letsencrypt.org/privacy".to_string()),
-Url::from("https://letsencrypt.org/sr".to_string()),
-Url::from("https://letsencrypt.org/id".to_string()),
-Url::from("https://letsencrypt.org/#".to_string()),
-Url::from("https://letsencrypt.org/docs".to_string()),
-Url::from("https://letsencrypt.org/sr/stats".to_string()),
-Url::from("https://letsencrypt.org/hu".to_string()),
-Url::from("https://letsencrypt.org/he/stats".to_string()),
-Url::from("https://letsencrypt.org/sponsors".to_string()),
-Url::from("https://letsencrypt.org/ja/stats".to_string()),
-Url::from("https://letsencrypt.org/getting-started".to_string()),
-Url::from("https://letsencrypt.org/de/stats".to_string()),
-Url::from("https://letsencrypt.org/docs/dst-root-ca-x3-expiration-september-2021".to_string()),
-Url::from("https://letsencrypt.org/sv/stats".to_string()),
-Url::from("https://letsencrypt.org/hu/stats".to_string()),
-Url::from("https://letsencrypt.org/es".to_string()),
-Url::from("https://letsencrypt.org/it".to_string()),
-Url::from("https://letsencrypt.org/it/stats".to_string()),
-Url::from("https://letsencrypt.org/zh-cn/stats".to_string()),
-Url::from("https://letsencrypt.org/sv".to_string()),
-Url::from("https://letsencrypt.org/getinvolved".to_string()),
-Url::from("https://letsencrypt.org/ru".to_string()),
-Url::from("https://letsencrypt.org/repository".to_string()),
-Url::from("https://letsencrypt.org/zh-cn".to_string()),
-Url::from("https://letsencrypt.org/fr".to_string()),
-Url::from("https://letsencrypt.org/da".to_string()),
-Url::from("https://letsencrypt.org/about".to_string()),
-Url::from("https://letsencrypt.org/ru/stats".to_string()),
-Url::from("https://letsencrypt.org/ko/stats".to_string()),
-Url::from("https://letsencrypt.org/stats/#percent-pageloads".to_string()),
-Url::from("https://letsencrypt.org/ko".to_string()),
-Url::from("https://letsencrypt.org/2022/12/15/ocspcaching.html".to_string()),
-Url::from("https://community.letsencrypt.org".to_string()),
-Url::from("https://letsencrypt.org/si/stats".to_string()),
-Url::from("https://letsencrypt.org/es/stats".to_string()),
-Url::from("https://letsencrypt.org/pt-br/stats".to_string()),
-Url::from("https://letsencrypt.org/uk".to_string()),
-Url::from("https://letsencrypt.org/de".to_string()),
-Url::from("https://letsencrypt.org/donate".to_string()),
-Url::from("https://letsencrypt.org/fi/stats".to_string()),
-Url::from("https://letsencrypt.org/2023/01/19/renewing-sponsors.html".to_string()),
-Url::from("https://letsencrypt.org/ja".to_string()),
-Url::from("https://letsencrypt.org/da/stats".to_string()),
-Url::from("https://letsencrypt.org/2023/01/12/eng-culture-at-isrg.html".to_string()),
-Url::from("https://letsencrypt.org/id/stats".to_string()),
-Url::from("https://letsencrypt.org/zh-tw/stats".to_string()),
-Url::from("https://letsencrypt.org/2023/05/24/isrg-10th-anniversary.html".to_string()),
+Url::parse("https://letsencrypt.org/2019/11/20/how-le-runs-ct-logs.html").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/invalid-certificate-on-mobile-works-on-desktop/24295").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/frequently-asked-questions-faq").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/upcoming-intermediate-changes/13106").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/san-domain-name-mismatch-android-2-windows-xp/4060").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/small-change-to-end-entity-certificates-cps-url-and-oid-will-not-be-included-from-june-15/198206").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/web-hosting-who-support-lets-encrypt/6920").unwrap(),
+Url::parse("https://letsencrypt.org/2022/04/13/receiving-the-levchin-prize.html").unwrap(),
+Url::parse("https://letsencrypt.org/2020/02/27/one-billion-certs.html").unwrap(),
+Url::parse("https://letsencrypt.org/docs/integration-guide").unwrap(),
+Url::parse("https://letsencrypt.org/fr/docs").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-v3.4").unwrap(),
+Url::parse("https://letsencrypt.org/sr/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/id/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/de/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/he/about").unwrap(),
+Url::parse("https://letsencrypt.org/da/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/es/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/da/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/blog/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/documents/LE-SA-v1.0.1-v1.1.1-diff.docx").unwrap(),
+Url::parse("https://letsencrypt.org/ja/#").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/2015/02/19/draft-isrg-cp.html").unwrap(),
+Url::parse("https://letsencrypt.org/2021/10/21/celebrating-encryption-globally.html").unwrap(),
+Url::parse("https://letsencrypt.org/docs/lencr.org").unwrap(),
+Url::parse("https://letsencrypt.org/it/repository").unwrap(),
+Url::parse("https://community.letsencrypt.org/guidelines").unwrap(),
+Url::parse("https://letsencrypt.org/vi/donate").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/sponsors/#").unwrap(),
+Url::parse("https://letsencrypt.org/fi/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/de/docs").unwrap(),
+Url::parse("https://letsencrypt.org/vi/docs").unwrap(),
+Url::parse("https://letsencrypt.org/docs/glossary/#def-root").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/KOKU9408").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/unauthorized-certificate-issued-for-domain/200845").unwrap(),
+Url::parse("https://letsencrypt.org/contact/#").unwrap(),
+Url::parse("https://letsencrypt.org/2017/10/17/acme-support-in-apache-httpd.html").unwrap(),
+Url::parse("https://letsencrypt.org/fr/donate").unwrap(),
+Url::parse("https://letsencrypt.org/docs/ipv6-support").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/frederik.lohse").unwrap(),
+Url::parse("https://letsencrypt.org/ko/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.3").unwrap(),
+Url::parse("https://letsencrypt.org/fi/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/docs/certificates-for-localhost").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/Oosterkamp").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v3.2").unwrap(),
+Url::parse("https://letsencrypt.org/si/sponsors").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/ssl-certificate-stopped-working/200817").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/dns-dnssec-error-signal/200808").unwrap(),
+Url::parse("https://letsencrypt.org/da/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/docs/staging-environment").unwrap(),
+Url::parse("https://letsencrypt.org/docs/acme-protocol-updates").unwrap(),
+Url::parse("https://letsencrypt.org/si/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/repository").unwrap(),
+Url::parse("https://letsencrypt.org/documents/LE-SA-v1.1.1-v1.2-diff.docx").unwrap(),
+Url::parse("https://letsencrypt.org/de/#").unwrap(),
+Url::parse("https://letsencrypt.org/it/contact").unwrap(),
+Url::parse("https://letsencrypt.org/2016/09/20/what-it-costs-to-run-lets-encrypt.html").unwrap(),
+Url::parse("https://letsencrypt.org/it/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/docs/integration-guide/#when-to-renew").unwrap(),
+Url::parse("https://letsencrypt.org/ko/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/fr/#").unwrap(),
+Url::parse("https://letsencrypt.org/ru/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/hu/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/2015/11/12/public-beta-timing.html/#").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/chandresh").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/apache2-server-certbot-unable-to-find-corresponding-http-vhost/200854").unwrap(),
+Url::parse("https://letsencrypt.org/2015/10/08/isoc-sponsors-lets-encrypt.html").unwrap(),
+Url::parse("https://letsencrypt.org/ja/repository").unwrap(),
+Url::parse("https://letsencrypt.org/fi/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/sr/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/vi/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/2021/02/10/200m-certs-24hrs.html").unwrap(),
+Url::parse("https://letsencrypt.org/hu/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/fr/contact").unwrap(),
+Url::parse("https://letsencrypt.org/docs/too-many-registrations-for-this-ip").unwrap(),
+Url::parse("https://letsencrypt.org/he/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/si/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/sr/stats/#").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/schoen").unwrap(),
+Url::parse("https://letsencrypt.org/2023/01/12/eng-culture-at-isrg.html/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v4.2").unwrap(),
+Url::parse("https://letsencrypt.org/ru/contact").unwrap(),
+Url::parse("https://letsencrypt.org/2014/11/18/announcing-lets-encrypt.html").unwrap(),
+Url::parse("https://letsencrypt.org/id/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/2016/03/09/le-client-new-home.html").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/2015/10/29/phishing-and-malware.html").unwrap(),
+Url::parse("https://letsencrypt.org/id/docs").unwrap(),
+Url::parse("https://letsencrypt.org/ko/sponsors").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/biwek86619").unwrap(),
+Url::parse("https://letsencrypt.org/docs/certificate-compatibility").unwrap(),
+Url::parse("https://letsencrypt.org/fi/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/he/docs").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-v3.2").unwrap(),
+Url::parse("https://letsencrypt.org/ko/stats/#").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/new-certficiate-not-point-to-the-file/200732").unwrap(),
+Url::parse("https://letsencrypt.org/de/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/ja/donate").unwrap(),
+Url::parse("https://letsencrypt.org/de/getinvolved").unwrap(),
+Url::parse("https://community.letsencrypt.org/c/help/ajuda-em-portugues").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/2018/04/04/sct-encoding.html").unwrap(),
+Url::parse("https://letsencrypt.org/about/#").unwrap(),
+Url::parse("https://community.letsencrypt.org/latest?no_definitions=true&page=1").unwrap(),
+Url::parse("https://letsencrypt.org/hu/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/id/#").unwrap(),
+Url::parse("https://letsencrypt.org/hu/contact").unwrap(),
+Url::parse("https://letsencrypt.org/fi/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/he/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/docs/#").unwrap(),
+Url::parse("https://letsencrypt.org/ko/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/uk/#").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/is-it-wise-to-use-lets-encrypt-certificates-for-a-production-endpoint/200836").unwrap(),
+Url::parse("https://letsencrypt.org/he/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/es/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/ko/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/Nummer378").unwrap(),
+Url::parse("https://letsencrypt.org/fi/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/fi/repository").unwrap(),
+Url::parse("https://letsencrypt.org/2023/03/23/improving-resliiency-and-reliability-with-ari.html/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/#").unwrap(),
+Url::parse("https://letsencrypt.org/he/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/de/contact").unwrap(),
+Url::parse("https://letsencrypt.org/si/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/2020/11/06/own-two-feet.html").unwrap(),
+Url::parse("https://letsencrypt.org/he/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/ja/docs").unwrap(),
+Url::parse("https://community.letsencrypt.org/categories").unwrap(),
+Url::parse("https://letsencrypt.org/docs/a-warm-welcome-to-asn1-and-der").unwrap(),
+Url::parse("https://letsencrypt.org/sv/getting-started").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/josh").unwrap(),
+Url::parse("https://letsencrypt.org/sr/#").unwrap(),
+Url::parse("https://letsencrypt.org/vi/privacy").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/0xffcaee").unwrap(),
+Url::parse("https://letsencrypt.org/da/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/repository").unwrap(),
+Url::parse("https://letsencrypt.org/id/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/he/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/2016/08/05/le-root-to-be-trusted-by-mozilla.html").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/jiapei100").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/jrsyo").unwrap(),
+Url::parse("https://letsencrypt.org/ko/contact").unwrap(),
+Url::parse("https://letsencrypt.org/2022/12/05/ed-letter-2022.html").unwrap(),
+Url::parse("https://letsencrypt.org/docs/rate-limits").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/greatlakesclub").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/o-t-1-options-in-access-log/200818").unwrap(),
+Url::parse("https://letsencrypt.org/ru/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/docs").unwrap(),
+Url::parse("https://letsencrypt.org/si/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/docs/ct-logs").unwrap(),
+Url::parse("https://letsencrypt.org/sv/repository").unwrap(),
+Url::parse("https://letsencrypt.org/2016/11/01/launching-our-crowdfunding-campaign.html").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.7").unwrap(),
+Url::parse("https://letsencrypt.org/de/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/es/donate").unwrap(),
+Url::parse("https://letsencrypt.org/da/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/contact").unwrap(),
+Url::parse("https://letsencrypt.org/it/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/2016/04/01/legal-transparency-report.html").unwrap(),
+Url::parse("https://letsencrypt.org/it/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/fr/stats/#growth").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/openssl-client-compatibility-changes-for-let-s-encrypt-certificates/143816").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.5").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/2017/07/06/wildcard-certificates-coming-jan-2018.html").unwrap(),
+Url::parse("https://letsencrypt.org/fr/about").unwrap(),
+Url::parse("https://letsencrypt.org/de/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/uk/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/2022/09/12/remembering-peter-eckersley.html").unwrap(),
+Url::parse("https://letsencrypt.org/ru/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/2016/04/12/leaving-beta-new-sponsors.html").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/some-vn-cctld-sld-arent-recognized-and-causing-ratelimited-error/200596").unwrap(),
+Url::parse("https://letsencrypt.org/ko/donate").unwrap(),
+Url::parse("https://letsencrypt.org/docs/duplicate-certificate-limit").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/2023/01/12/eng-culture-at-isrg.html/#").unwrap(),
+Url::parse("https://letsencrypt.org/2020/12/21/extending-android-compatibility.html").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/Rip").unwrap(),
+Url::parse("https://letsencrypt.org/de/donate").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/rmhrisk").unwrap(),
+Url::parse("https://community.letsencrypt.org/c/help/ayuda-en-espanol/22").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/acme-v2-production-environment-wildcards/55578").unwrap(),
+Url::parse("https://letsencrypt.org/vi/stats/#").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/tls-1-2-and-tls-1-3-need-curve25519-and-curve448-ssl-certificates/200775").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/dns-problem-query-timed-out-looking-up-a/200751").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/Nekit").unwrap(),
+Url::parse("https://letsencrypt.org/privacy/#").unwrap(),
+Url::parse("https://letsencrypt.org/fi/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/vi/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/ru/donate").unwrap(),
+Url::parse("https://letsencrypt.org/da/#").unwrap(),
+Url::parse("https://letsencrypt.org/vi/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v3.1").unwrap(),
+Url::parse("https://letsencrypt.org/2015/08/13/lets-encrypt-community-support.html").unwrap(),
+Url::parse("https://letsencrypt.org/vi/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/fr/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/ru/docs/faq").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/is-there-anything-new-to-generate-lets-encrypt-certificate-with-docker-and-nginx/200837").unwrap(),
+Url::parse("https://letsencrypt.org/uk/docs").unwrap(),
+Url::parse("https://letsencrypt.org/si/about").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/certbot-renew-task-windows-setting/200282").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/keeptake").unwrap(),
+Url::parse("https://letsencrypt.org/vi/about").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/docs").unwrap(),
+Url::parse("https://letsencrypt.org/docs/faq/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.1").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/Bruce5051").unwrap(),
+Url::parse("https://letsencrypt.org/si/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/ru/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/sr/about").unwrap(),
+Url::parse("https://letsencrypt.org/da/getinvolved").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/why-i-cant-acess-this-site/200850").unwrap(),
+Url::parse("https://letsencrypt.org/2015/12/03/facebook-sponsorship.html").unwrap(),
+Url::parse("https://cp.letsencrypt.org").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/docs").unwrap(),
+Url::parse("https://letsencrypt.org/id/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.3").unwrap(),
+Url::parse("https://letsencrypt.org/fr/repository").unwrap(),
+Url::parse("https://letsencrypt.org/sv/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/2015/11/12/public-beta-timing.html/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/donate").unwrap(),
+Url::parse("https://letsencrypt.org/ja/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/sr/docs/faq").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/jacordar").unwrap(),
+Url::parse("https://letsencrypt.org/sr/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/es/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/2017/01/06/le-2016-in-review.html").unwrap(),
+Url::parse("https://letsencrypt.org/2022/02/25/ford-foundation.html").unwrap(),
+Url::parse("https://letsencrypt.org/2023/03/23/improving-resliiency-and-reliability-with-ari.html/#").unwrap(),
+Url::parse("https://letsencrypt.org/si/#").unwrap(),
+Url::parse("https://letsencrypt.org/docs/caa").unwrap(),
+Url::parse("https://letsencrypt.org/es/docs").unwrap(),
+Url::parse("https://letsencrypt.org/2016/10/01/legal-transparency-report.html").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.6").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/#").unwrap(),
+Url::parse("https://letsencrypt.org/fi/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/hu/stats/#").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/mlen").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/about").unwrap(),
+Url::parse("https://letsencrypt.org/sv/about").unwrap(),
+Url::parse("https://letsencrypt.org/docs/dst-root-ca-x3-expiration-september-2021/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/ja/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/sv/#").unwrap(),
+Url::parse("https://letsencrypt.org/ru/docs").unwrap(),
+Url::parse("https://letsencrypt.org/docs/allow-port-80").unwrap(),
+Url::parse("https://letsencrypt.org/2015/06/04/isrg-ca-certs.html").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v4.5").unwrap(),
+Url::parse("https://letsencrypt.org/es/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/2015/12/03/entering-public-beta.html/#").unwrap(),
+Url::parse("https://letsencrypt.org/2016/10/21/introducing-idn-support.html").unwrap(),
+Url::parse("https://letsencrypt.org/uk/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.6").unwrap(),
+Url::parse("https://letsencrypt.org/certificates").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/thunderbird-exim-dovecot-cert-settings/200882").unwrap(),
+Url::parse("https://letsencrypt.org/fi/donate").unwrap(),
+Url::parse("https://letsencrypt.org/ru/sponsors").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/rahvee_tufts").unwrap(),
+Url::parse("https://letsencrypt.org/ko/docs").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-cps-v5.0").unwrap(),
+Url::parse("https://letsencrypt.org/ru/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/contact").unwrap(),
+Url::parse("https://letsencrypt.org/privacy/#main-content").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/lordspace").unwrap(),
+Url::parse("https://letsencrypt.org/docs/glossary").unwrap(),
+Url::parse("https://letsencrypt.org/sr/docs").unwrap(),
+Url::parse("https://letsencrypt.org/sv/docs").unwrap(),
+Url::parse("https://letsencrypt.org/2020/09/17/new-root-and-intermediates.html").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/he/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.5").unwrap(),
+Url::parse("https://letsencrypt.org/uk/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/fr/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/2015/06/16/lets-encrypt-launch-schedule.html").unwrap(),
+Url::parse("https://letsencrypt.org/uk/docs/faq").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/welcome-to-lets-encrypt-community-support/8").unwrap(),
+Url::parse("https://letsencrypt.org/2019/10/09/onboarding-your-customers-with-lets-encrypt-and-acme.html").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/no-renewal-config-found/200878").unwrap(),
+Url::parse("https://letsencrypt.org/it/donate").unwrap(),
+Url::parse("https://letsencrypt.org/he/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/id/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/hu/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v4.1").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/reminder-email-after-adding-or-removing-names/200846").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/dh-2048-to-dh-4096/200885").unwrap(),
+Url::parse("https://letsencrypt.org/hu/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/ko/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/docs/godaddy").unwrap(),
+Url::parse("https://letsencrypt.org/ja/about").unwrap(),
+Url::parse("https://letsencrypt.org/2021/10/28/tls-simply-and-automatically.html").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.0").unwrap(),
+Url::parse("https://letsencrypt.org/2018/12/31/looking-forward-to-2019.html").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/stats/#growth").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/kaponir").unwrap(),
+Url::parse("https://letsencrypt.org/sr/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/sr/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/vi/contact").unwrap(),
+Url::parse("https://letsencrypt.org/2016/10/27/first-grant-ford-foundation.html").unwrap(),
+Url::parse("https://letsencrypt.org/sv/stats/#main-content").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/multiple-ssl-with-a-single-mail-server/200804").unwrap(),
+Url::parse("https://letsencrypt.org/it/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/fi/#").unwrap(),
+Url::parse("https://letsencrypt.org/da/docs").unwrap(),
+Url::parse("https://community.letsencrypt.org/c/issuance-tech-questions/12").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/mastodon-type-unauthorized-invalid-response-from-404/200815").unwrap(),
+Url::parse("https://letsencrypt.org/docs/challenge-types/#dns-01-challenge").unwrap(),
+Url::parse("https://cps.letsencrypt.org").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v3.0").unwrap(),
+Url::parse("https://letsencrypt.org/he/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/fr/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/es/#").unwrap(),
+Url::parse("https://letsencrypt.org/uk/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/2018/08/06/trusted-by-all-major-root-programs.html").unwrap(),
+Url::parse("https://letsencrypt.org/id/about").unwrap(),
+Url::parse("https://letsencrypt.org/id/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/sv/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/2023/01/19/renewing-sponsors.html/#").unwrap(),
+Url::parse("https://letsencrypt.org/2019/05/15/introducing-oak-ct-log.html").unwrap(),
+Url::parse("https://letsencrypt.org/ja/stats/#growth").unwrap(),
+Url::parse("https://community.letsencrypt.org/latest.rss").unwrap(),
+Url::parse("https://letsencrypt.org/2015/04/09/isrg-lf-collaboration.html").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/fr/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/ru/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/he/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/docs/account-id").unwrap(),
+Url::parse("https://letsencrypt.org/da/about").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/da/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/de/about").unwrap(),
+Url::parse("https://letsencrypt.org/docs/challenge-types").unwrap(),
+Url::parse("https://letsencrypt.org/2015/08/07/updated-lets-encrypt-launch-schedule.html").unwrap(),
+Url::parse("https://letsencrypt.org/it/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/uk/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/hu/donate").unwrap(),
+Url::parse("https://letsencrypt.org/sr/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/fr/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/id/contact").unwrap(),
+Url::parse("https://letsencrypt.org/de/privacy").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/help-thread-for-dst-root-ca-x3-expiration-september-2021/149190").unwrap(),
+Url::parse("https://letsencrypt.org/ru/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/sv/donate").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.9").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/repository").unwrap(),
+Url::parse("https://letsencrypt.org/hu/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/hu/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/repository/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/ko/#").unwrap(),
+Url::parse("https://letsencrypt.org/2015/10/19/lets-encrypt-is-trusted.html").unwrap(),
+Url::parse("https://letsencrypt.org/sv/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v4.0").unwrap(),
+Url::parse("https://letsencrypt.org/he/stats/#").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/xtronics").unwrap(),
+Url::parse("https://letsencrypt.org/es/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/id/privacy").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/robouden").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/ko/#main-content").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/Osiris").unwrap(),
+Url::parse("https://letsencrypt.org/docs/#main-content").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/jvanasco").unwrap(),
+Url::parse("https://letsencrypt.org/2015/12/21/ovh-sponsorship.html").unwrap(),
+Url::parse("https://letsencrypt.org/si/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/es/contact").unwrap(),
+Url::parse("https://letsencrypt.org/es/about").unwrap(),
+Url::parse("https://letsencrypt.org/2016/03/08/our-millionth-cert.html").unwrap(),
+Url::parse("https://letsencrypt.org/docs/revoking").unwrap(),
+Url::parse("https://letsencrypt.org/si/repository").unwrap(),
+Url::parse("https://letsencrypt.org/de/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/vi/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/mholt").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-v3.0").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/renewal-not-avaiable/200869").unwrap(),
+Url::parse("https://letsencrypt.org/de/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/2019/04/15/transitioning-to-isrg-root.html").unwrap(),
+Url::parse("https://letsencrypt.org/2021/09/14/speed-at-scale-shopify.html").unwrap(),
+Url::parse("https://community.letsencrypt.org/c/client-dev/14").unwrap(),
+Url::parse("https://letsencrypt.org/getinvolved/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/ja/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/ru/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/id/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/it/#main-content").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/_az").unwrap(),
+Url::parse("https://letsencrypt.org/it/docs").unwrap(),
+Url::parse("https://letsencrypt.org/ko/repository").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/rg305").unwrap(),
+Url::parse("https://letsencrypt.org/id/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/2022/04/28/database-to-app-tls.html").unwrap(),
+Url::parse("https://letsencrypt.org/uk/trademarks").unwrap(),
+Url::parse("https://letsencrypt.org/sr/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/fi/about").unwrap(),
+Url::parse("https://letsencrypt.org/id/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/it/about").unwrap(),
+Url::parse("https://letsencrypt.org/fi/#main-content").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/linkp").unwrap(),
+Url::parse("https://community.letsencrypt.org/tos").unwrap(),
+Url::parse("https://letsencrypt.org/ja/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/upcoming-features").unwrap(),
+Url::parse("https://letsencrypt.org/uk/donate").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/aarongable").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/Mahmoud1").unwrap(),
+Url::parse("https://letsencrypt.org/sr/stats/#percent-pageloads").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/yyny").unwrap(),
+Url::parse("https://letsencrypt.org/ja/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/petercooperjr").unwrap(),
+Url::parse("https://letsencrypt.org/da/donate").unwrap(),
+Url::parse("https://letsencrypt.org/id/repository").unwrap(),
+Url::parse("https://letsencrypt.org/si/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/vi/#").unwrap(),
+Url::parse("https://letsencrypt.org/it/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/es/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/id/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/hu/docs").unwrap(),
+Url::parse("https://letsencrypt.org/ko/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/donate/#").unwrap(),
+Url::parse("https://letsencrypt.org/blog/#").unwrap(),
+Url::parse("https://letsencrypt.org/vi/stats/#growth").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/cocolinofan").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.0").unwrap(),
+Url::parse("https://letsencrypt.org/sv/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/fr/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/de/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/ru/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/si/docs").unwrap(),
+Url::parse("https://letsencrypt.org/getinvolved/#").unwrap(),
+Url::parse("https://letsencrypt.org/hu/#").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/2023/05/24/isrg-10th-anniversary.html/#").unwrap(),
+Url::parse("https://letsencrypt.org/2019/02/12/facebook-expands-support-for-letsencrypt.html").unwrap(),
+Url::parse("https://letsencrypt.org/hu/repository").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/production-chain-changes/150739/4").unwrap(),
+Url::parse("https://letsencrypt.org/docs/dst-root-ca-x3-expiration-september-2021/#").unwrap(),
+Url::parse("https://letsencrypt.org/ko/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/ja/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/2021/12/16/ed-letter-2021.html").unwrap(),
+Url::parse("https://letsencrypt.org/2017/03/23/ovh-platinum-renewal.html").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/smyasoedov").unwrap(),
+Url::parse("https://letsencrypt.org/es/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/2016/06/22/https-progress-june-2016.html").unwrap(),
+Url::parse("https://letsencrypt.org/2021/01/21/next-gen-database-servers.html").unwrap(),
+Url::parse("https://letsencrypt.org/da/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/fr/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/2016/07/26/full-ipv6-support.html").unwrap(),
+Url::parse("https://letsencrypt.org/he/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/2022/05/19/nurturing-ct-log-growth.html").unwrap(),
+Url::parse("https://letsencrypt.org/2015/07/01/legal-transparency-report.html").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.1").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/certbot-invalid-response-http-www-example-org-well-known-acme-challenge/200841").unwrap(),
+Url::parse("https://letsencrypt.org/hu/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/sr/repository").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/production-chain-changes/150739").unwrap(),
+Url::parse("https://letsencrypt.org/docs/faq/#").unwrap(),
+Url::parse("https://letsencrypt.org/es/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/vi/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/security_letsencrypt.org-publickey.asc").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/using-staging-and-certbot-saves-stuff-in-live-directory/200780").unwrap(),
+Url::parse("https://letsencrypt.org/2023/01/19/renewing-sponsors.html/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/contact/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/2015/04/14/ncc-group-audit.html").unwrap(),
+Url::parse("https://letsencrypt.org/it/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/ja/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/2017/12/07/looking-forward-to-2018.html").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/about").unwrap(),
+Url::parse("https://letsencrypt.org/sr/contact").unwrap(),
+Url::parse("https://letsencrypt.org/2015/11/09/why-90-days.html/#").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/uk/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-v3.3").unwrap(),
+Url::parse("https://letsencrypt.org/sr/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/si/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/2017/06/14/acme-v2-api.html").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/about").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-v3.1").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/hello-can-i-use-lets-encrypt-for-postfix/200169").unwrap(),
+Url::parse("https://letsencrypt.org/hu/about").unwrap(),
+Url::parse("https://letsencrypt.org/getting-started/#").unwrap(),
+Url::parse("https://letsencrypt.org/2015/12/03/entering-public-beta.html/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/sv/contact").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v4.4").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.2").unwrap(),
+Url::parse("https://letsencrypt.org/docs/failed-validation-limit").unwrap(),
+Url::parse("https://letsencrypt.org/sr/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/fr/stats/#").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/system").unwrap(),
+Url::parse("https://community.letsencrypt.org/c/issuance-policy/11").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/knX5Fw").unwrap(),
+Url::parse("https://letsencrypt.org/da/repository").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/acmeisuptime-com/200022").unwrap(),
+Url::parse("https://letsencrypt.org/ru/repository").unwrap(),
+Url::parse("https://letsencrypt.org/2015/05/21/draft-le-sa.html").unwrap(),
+Url::parse("https://letsencrypt.org/how-it-works").unwrap(),
+Url::parse("https://letsencrypt.org/vi/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/fi/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.4").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/uk/contact").unwrap(),
+Url::parse("https://letsencrypt.org/fr/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/it/#").unwrap(),
+Url::parse("https://letsencrypt.org/2023/05/24/isrg-10th-anniversary.html/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/vi/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/2015/09/14/our-first-cert.html").unwrap(),
+Url::parse("https://letsencrypt.org/es/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.4").unwrap(),
+Url::parse("https://letsencrypt.org/ko/about").unwrap(),
+Url::parse("https://letsencrypt.org/he/contact").unwrap(),
+Url::parse("https://letsencrypt.org/uk/about").unwrap(),
+Url::parse("https://letsencrypt.org/it/stats/#main-content").unwrap(),
+Url::parse("https://community.letsencrypt.org/c/help/aide-en-francais").unwrap(),
+Url::parse("https://letsencrypt.org/sponsors/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/sv/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v3.3").unwrap(),
+Url::parse("https://letsencrypt.org/he/#").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-cps-v5.1").unwrap(),
+Url::parse("https://letsencrypt.org/fi/stats/#percent-pageloads").unwrap(),
+Url::parse("https://community.letsencrypt.org/posts.rss").unwrap(),
+Url::parse("https://letsencrypt.org/documents/LE-SA-v1.2-v1.3-diff.docx").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/chit164").unwrap(),
+Url::parse("https://letsencrypt.org/da/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/uk/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/si/contact").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/stats/#daily-issuance").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/renew-dry-run-fails-with-unable-to-determine-base-domain-error/200831").unwrap(),
+Url::parse("https://letsencrypt.org/sv/stats/#percent-pageloads").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/webprofusion").unwrap(),
+Url::parse("https://letsencrypt.org/uk/repository").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.2").unwrap(),
+Url::parse("https://letsencrypt.org/uk/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/ja/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/2016/10/24/squarespace-ocsp-impl.html").unwrap(),
+Url::parse("https://letsencrypt.org/de/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/2016/06/23/defending-our-brand.html").unwrap(),
+Url::parse("https://letsencrypt.org/2015/04/23/draft-isrg-cp-cps.html").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/orangepizza").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/problema-de-conexion-connection-problem/200673").unwrap(),
+Url::parse("https://letsencrypt.org/id/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/docs/expiration-emails").unwrap(),
+Url::parse("https://letsencrypt.org/it/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/uk/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/es/repository").unwrap(),
+Url::parse("https://letsencrypt.org/donate/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/de/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/donate").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cp-v2.7").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/JoseGR02").unwrap(),
+Url::parse("https://letsencrypt.org/2020/12/28/executive-director-letter.html").unwrap(),
+Url::parse("https://letsencrypt.org/ja/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/contact").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/dns-01-challenge-start-to-failure-since-24-at-22h/200816").unwrap(),
+Url::parse("https://letsencrypt.org/ru/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/ko/stats/#percent-pageloads").unwrap(),
+Url::parse("https://community.letsencrypt.org/opensearch.xml").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/fi/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/hu/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/2020/02/19/multi-perspective-validation.html").unwrap(),
+Url::parse("https://letsencrypt.org/docs/client-options").unwrap(),
+Url::parse("https://letsencrypt.org/uk/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/getting-started/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/repository/#").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v2.8").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/thoughts-from-starting-to-play-with-ari/200276").unwrap(),
+Url::parse("https://letsencrypt.org/ru/#").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/Krischu").unwrap(),
+Url::parse("https://letsencrypt.org/sr/donate").unwrap(),
+Url::parse("https://letsencrypt.org/docs/faq/#technical").unwrap(),
+Url::parse("https://letsencrypt.org/si/stats/#main-content").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/vrubiella").unwrap(),
+Url::parse("https://letsencrypt.org/fi/contact").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/stats/#main-content").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/MikeMcQ").unwrap(),
+Url::parse("https://letsencrypt.org/es/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/he/repository").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/mcpherrinm").unwrap(),
+Url::parse("https://letsencrypt.org/vi/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/id/donate").unwrap(),
+Url::parse("https://letsencrypt.org/da/stats/#daily-issuance").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/phonegal2").unwrap(),
+Url::parse("https://letsencrypt.org/sv/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/si/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/2019/03/11/acme-protocol-ietf-standard.html").unwrap(),
+Url::parse("https://letsencrypt.org/si/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/fi/docs").unwrap(),
+Url::parse("https://letsencrypt.org/2022/12/15/ocspcaching.html/#").unwrap(),
+Url::parse("https://letsencrypt.org/da/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/he/donate").unwrap(),
+Url::parse("https://letsencrypt.org/docs/faq/#general").unwrap(),
+Url::parse("https://letsencrypt.org/fr/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/2015/11/09/why-90-days.html/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/fr/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/donate").unwrap(),
+Url::parse("https://letsencrypt.org/vi/repository").unwrap(),
+Url::parse("https://letsencrypt.org/ja/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/ko/stats/#main-content").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/mason8592").unwrap(),
+Url::parse("https://letsencrypt.org/2022/09/07/new-life-for-crls.html").unwrap(),
+Url::parse("https://letsencrypt.org/2022/12/15/ocspcaching.html/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/ja/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/2017/06/28/hundred-million-certs.html").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/uk/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/da/contact").unwrap(),
+Url::parse("https://letsencrypt.org/hu/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/de/repository").unwrap(),
+Url::parse("https://letsencrypt.org/it/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/es/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/it/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/da/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/es/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/hu/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/si/donate").unwrap(),
+Url::parse("https://letsencrypt.org/about/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/de/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/ru/about").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/letsuser").unwrap(),
+Url::parse("https://community.letsencrypt.org/u/nigihayami").unwrap(),
+Url::parse("https://letsencrypt.org/sv/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/sv/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://community.letsencrypt.org/c/help/13").unwrap(),
+Url::parse("https://letsencrypt.org/sv/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/documents/isrg-cps-v4.3").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/#").unwrap(),
+Url::parse("https://letsencrypt.org/trademarks").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/2021/10/01/cert-chaining-help.html").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/get-error-ref-5401-then-now-ref-5017/200867").unwrap(),
+Url::parse("https://letsencrypt.org/ja/contact").unwrap(),
+Url::parse("https://www.letsencrypt.org").unwrap(),
+Url::parse("https://letsencrypt.org/ja/docs/allow-port-80").unwrap(),
+Url::parse("https://letsencrypt.org/docs/rate-limits#:~:text=limit%20of%205%20per%20week").unwrap(),
+Url::parse("https://letsencrypt.org/docs/faq/#what-ip-addresses-does-let-s-encrypt-use-to-validate-my-web-server").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/revoking-certain-certificates-on-march-4/114864").unwrap(),
+Url::parse("https://letsencrypt.org/become-a-sponsor").unwrap(),
+Url::parse("https://letsencrypt.org/?ref=blog.sean-wright.com").unwrap(),
+Url::parse("https://letsencrypt.org/?ref=spreadprivacy.com").unwrap(),
+Url::parse("https://letsencrypt.org/howitworks/technology").unwrap(),
+Url::parse("https://community.letsencrypt.org/t/beta-program-announcements/1631/6").unwrap(),
+Url::parse("https://acme-v01.api.letsencrypt.org/directory").unwrap(),
+Url::parse("https://acme-v01.api.letsencrypt.org/directory%22").unwrap(),
+Url::parse("https://letsencrypt.org/feed.xml").unwrap(),
+Url::parse("https://letsencrypt.org/stats").unwrap(),
+Url::parse("https://letsencrypt.org").unwrap(),
+Url::parse("https://letsencrypt.org/2015/11/12/public-beta-timing.html").unwrap(),
+Url::parse("https://letsencrypt.org/2015/12/03/entering-public-beta.html").unwrap(),
+Url::parse("https://letsencrypt.org/2015/11/09/why-90-days.html").unwrap(),
+Url::parse("https://letsencrypt.org/2023/03/23/improving-resliiency-and-reliability-with-ari.html").unwrap(),
+Url::parse("https://letsencrypt.org/vi").unwrap(),
+Url::parse("https://letsencrypt.org/blog").unwrap(),
+Url::parse("https://letsencrypt.org/stats/#growth").unwrap(),
+Url::parse("https://letsencrypt.org/si").unwrap(),
+Url::parse("https://letsencrypt.org/docs/faq").unwrap(),
+Url::parse("https://letsencrypt.org/uk/stats").unwrap(),
+Url::parse("https://letsencrypt.org/contact").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw").unwrap(),
+Url::parse("https://letsencrypt.org/he").unwrap(),
+Url::parse("https://letsencrypt.org/stats/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/fi").unwrap(),
+Url::parse("https://letsencrypt.org/stats/#daily-issuance").unwrap(),
+Url::parse("https://letsencrypt.org/vi/stats").unwrap(),
+Url::parse("https://letsencrypt.org/fr/stats").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br").unwrap(),
+Url::parse("https://letsencrypt.org/stats/#").unwrap(),
+Url::parse("https://letsencrypt.org/#main-content").unwrap(),
+Url::parse("https://letsencrypt.org/privacy").unwrap(),
+Url::parse("https://letsencrypt.org/sr").unwrap(),
+Url::parse("https://letsencrypt.org/id").unwrap(),
+Url::parse("https://letsencrypt.org/#").unwrap(),
+Url::parse("https://letsencrypt.org/docs").unwrap(),
+Url::parse("https://letsencrypt.org/sr/stats").unwrap(),
+Url::parse("https://letsencrypt.org/hu").unwrap(),
+Url::parse("https://letsencrypt.org/he/stats").unwrap(),
+Url::parse("https://letsencrypt.org/sponsors").unwrap(),
+Url::parse("https://letsencrypt.org/ja/stats").unwrap(),
+Url::parse("https://letsencrypt.org/getting-started").unwrap(),
+Url::parse("https://letsencrypt.org/de/stats").unwrap(),
+Url::parse("https://letsencrypt.org/docs/dst-root-ca-x3-expiration-september-2021").unwrap(),
+Url::parse("https://letsencrypt.org/sv/stats").unwrap(),
+Url::parse("https://letsencrypt.org/hu/stats").unwrap(),
+Url::parse("https://letsencrypt.org/es").unwrap(),
+Url::parse("https://letsencrypt.org/it").unwrap(),
+Url::parse("https://letsencrypt.org/it/stats").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn/stats").unwrap(),
+Url::parse("https://letsencrypt.org/sv").unwrap(),
+Url::parse("https://letsencrypt.org/getinvolved").unwrap(),
+Url::parse("https://letsencrypt.org/ru").unwrap(),
+Url::parse("https://letsencrypt.org/repository").unwrap(),
+Url::parse("https://letsencrypt.org/zh-cn").unwrap(),
+Url::parse("https://letsencrypt.org/fr").unwrap(),
+Url::parse("https://letsencrypt.org/da").unwrap(),
+Url::parse("https://letsencrypt.org/about").unwrap(),
+Url::parse("https://letsencrypt.org/ru/stats").unwrap(),
+Url::parse("https://letsencrypt.org/ko/stats").unwrap(),
+Url::parse("https://letsencrypt.org/stats/#percent-pageloads").unwrap(),
+Url::parse("https://letsencrypt.org/ko").unwrap(),
+Url::parse("https://letsencrypt.org/2022/12/15/ocspcaching.html").unwrap(),
+Url::parse("https://community.letsencrypt.org").unwrap(),
+Url::parse("https://letsencrypt.org/si/stats").unwrap(),
+Url::parse("https://letsencrypt.org/es/stats").unwrap(),
+Url::parse("https://letsencrypt.org/pt-br/stats").unwrap(),
+Url::parse("https://letsencrypt.org/uk").unwrap(),
+Url::parse("https://letsencrypt.org/de").unwrap(),
+Url::parse("https://letsencrypt.org/donate").unwrap(),
+Url::parse("https://letsencrypt.org/fi/stats").unwrap(),
+Url::parse("https://letsencrypt.org/2023/01/19/renewing-sponsors.html").unwrap(),
+Url::parse("https://letsencrypt.org/ja").unwrap(),
+Url::parse("https://letsencrypt.org/da/stats").unwrap(),
+Url::parse("https://letsencrypt.org/2023/01/12/eng-culture-at-isrg.html").unwrap(),
+Url::parse("https://letsencrypt.org/id/stats").unwrap(),
+Url::parse("https://letsencrypt.org/zh-tw/stats").unwrap(),
+Url::parse("https://letsencrypt.org/2023/05/24/isrg-10th-anniversary.html").unwrap(),
];
let apple_urls = vec![
-Url::from("https://www.apple.com/newsroom/2023/06/developer-tools-to-create-spatial-experiences-for-apple-vision-pro-now-available".to_string()),
-Url::from("https://apps.apple.com/us/app/top10-com/id1545027369".to_string()),
-Url::from("https://itunes.apple.com/us/app/safari-to-go/id881697395".to_string()),
-Url::from("https://support.apple.com/fr-fr/safari".to_string()),
-Url::from("https://apps.apple.com/app/id1145737091".to_string()),
-Url::from("https://itunes.apple.com/app/id473512078".to_string()),
-Url::from("https://itunes.apple.com/us/app/articles/id364881979".to_string()),
-Url::from("https://itunes.apple.com/us/app/geosearch-for-wikipedia/id929984054".to_string()),
-Url::from("https://itunes.apple.com/eg/app/wikiflip/id1156078290?mt=8".to_string()),
-Url::from("https://itunes.apple.com/app/wikiwand-faster-wikipedia-reader/id947444965".to_string()),
-Url::from("https://itunes.apple.com/us/app/learn-discovery-mindmap-wikipedia/id859954749".to_string()),
-Url::from("https://apps.apple.com/us/app/duovocal-learn-german/id1548499329".to_string()),
-Url::from("https://itunes.apple.com/app/viki-wikipedia/id993435362".to_string()),
-Url::from("https://itunes.apple.com/us/app/wikimed/id1281693200?mt=8".to_string()),
-Url::from("https://apps.apple.com/us/app/miniwiki/id1448143759#?platform=appleWatch".to_string()),
-Url::from("https://itunes.apple.com/app/wikicompass/id1363789167".to_string()),
-Url::from("https://itunes.apple.com/us/app/inquire-by-tamper/id1045268097".to_string()),
-Url::from("https://itunes.apple.com/us/app/kiwix/id997079563".to_string()),
-Url::from("https://itunes.apple.com/us/app/wikinodes/id433834594".to_string()),
-Url::from("https://itunes.apple.com/us/app/id1046122820".to_string()),
-Url::from("https://itunes.apple.com/en/app/wikipanion/id288349436".to_string()),
-Url::from("https://itunes.apple.com/us/app/wikiweb-visual-wikipedia-reader/id499345601".to_string()),
-Url::from("https://itunes.apple.com/eg/app/wikimaster/id1062887157?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/tech-xplore/id1154953172?mt=8".to_string()),
-Url::from("https://developer.apple.com/documentation/xcode/writing-armv6-code-for-ios".to_string()),
-Url::from("https://apps.apple.com/us/app/polygit-git-client/id1508942822".to_string()),
-Url::from("https://apps.apple.com/ro/app/cinemagia-mobile/id445358827".to_string()),
-Url::from("https://itunes.apple.com/jp/app/id1082193172?mt=8".to_string()),
-Url::from("https://help.apple.com/systemimageutility/mac/10.12/#/sysmfd460053".to_string()),
-Url::from("https://support.apple.com/en-gb/HT208050".to_string()),
-Url::from("https://developer.apple.com/documentation/swiftdata/relationship".to_string()),
-Url::from("https://developer.apple.com/documentation/SwiftData".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/when-to-use-microservices-when-not-to-sam-newman-martin/id1559093928?i=1000514252180".to_string()),
-Url::from("http://support.apple.com/kb/HT5012".to_string()),
-Url::from("http://www.apple.com/certificateauthority/ca_program.html".to_string()),
-Url::from("http://support.apple.com/kb/HT6005".to_string()),
-Url::from("http://www.apple.com".to_string()),
-Url::from("http://support.apple.com/kb/HT1677?viewlocale=da_DK".to_string()),
-Url::from("http://maps.apple.com/?q=Highgate+Cemetery,+Swains+Lane,+London+N6+6PJ".to_string()),
-Url::from("http://itunes.apple.com/us/book/isbn9780990582915".to_string()),
-Url::from("https://itunes.apple.com/us/app/git-drive/id1215974700?ls=1&mt=8".to_string()),
-Url::from("https://apple.com".to_string()),
-Url::from("https://www.apple.com/ca/osx/apps/app-store/index.html".to_string()),
-Url::from("https://support.apple.com/de-de/guide/safari/sfri11471/mac".to_string()),
-Url::from("https://support.apple.com/sv-se/guide/iphone/iph96b214f0/ios".to_string()),
-Url::from("https://itunes.apple.com/us/app/wikipedia-mobile/id324715238?mt=8".to_string()),
-Url::from("https://testflight.apple.com/join/Z0AU0KXC".to_string()),
-Url::from("https://itunes.apple.com/us/institution/library-of-congress/id386017159".to_string()),
-Url::from("https://itunes.apple.com/us/artist/library-of-congress/id1280713133?mt=2".to_string()),
-Url::from("https://itunes.apple.com/us/app/new-york-post-ipad-edition/id378590820?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/new-york-post-iphone-edition/id410094240?mt=8".to_string()),
-Url::from("https://itunes.apple.com/au/app/pocket-casts/id414834813".to_string()),
-Url::from("https://www.apple.com/apple-watch-ultra".to_string()),
-Url::from("http://support.apple.com/kb/PH5042".to_string()),
-Url::from("http://www.apple.com/macos".to_string()),
-Url::from("https://support.apple.com/en-us/HT208331".to_string()),
-Url::from("https://support.apple.com/en-us/HT205030".to_string()),
-Url::from("https://itunes.apple.com/us/artist/id2080713".to_string()),
-Url::from("https://www.apple.com/osx".to_string()),
-Url::from("http://support.apple.com/kb/HT4228".to_string()),
-Url::from("https://support.apple.com/en-ie/guide/safari/sfri11471/mac".to_string()),
-Url::from("https://support.apple.com/en-us/HT201265".to_string()),
-Url::from("http://itunes.apple.com/us/app/whatsapp-messenger/id310633997?mt=8&fbclid=IwAR30kILqJposAMfPq2JiGQ00o5NwVDc3jvu86wtcmAwzPNCTJXLZaooUpVk".to_string()),
-Url::from("https://support.apple.com/downloads/#safari".to_string()),
-Url::from("https://itunes.apple.com/us/app/the-identifiers/id564618183".to_string()),
-Url::from("https://apps.apple.com/app/packt/id1376577776?ls=1".to_string()),
-Url::from("https://appleid.apple.com/auth/authorize?redirect_uri=https%3A%2F%2Fwww.meetup.com%2Fties2%2F&state=returnUri%3Dhttps%253A%252F%252Fwww.meetup.com%252Fhome&client_id=com.meetup.web&response_type=code%20id_token&scope=name%20email&response_mode=form_post".to_string()),
-Url::from("https://apps.apple.com/us/app/meetup/id375990038".to_string()),
-Url::from("https://apple.com/mac".to_string()),
-Url::from("https://apps.apple.com/us/app/tamtam-messenger-video-calls/id1095345669".to_string()),
-Url::from("https://apps.apple.com/us/app/nicegram-ai-chat-for-telegram/id1608870673".to_string()),
-Url::from("https://apps.apple.com/app/telegram-messenger/id686449807?platform=iphone".to_string()),
-Url::from("https://apps.apple.com/us/developer/telegram-fz-llc/id686450210".to_string()),
-Url::from("https://support.apple.com/kb/PH2701".to_string()),
-Url::from("https://apps.apple.com/us/app/status-saver-video-photo-save/id1528272582".to_string()),
-Url::from("https://itunes.apple.com/us/genre/id6005".to_string()),
-Url::from("https://apps.apple.com/us/app/telegram-tools/id1621317550".to_string()),
-Url::from("https://apps.apple.com/us/app/telegram-messenger/id686449807".to_string()),
-Url::from("https://apps.apple.com/us/charts/iphone/social-networking-apps/6005".to_string()),
-Url::from("https://apps.apple.com/us/app/viber-messenger-chats-calls/id382617920".to_string()),
-Url::from("https://apps.apple.com/app/telegram-messenger/id686449807/www.apple.com/us/shop/goto/exitstore".to_string()),
-Url::from("https://apps.apple.com/app/telegram-messenger/id686449807/www.apple.com/us/shop/goto/home".to_string()),
-Url::from("https://apps.apple.com/us/app/dual-messenger-plus/id1530021020".to_string()),
-Url::from("https://apps.apple.com/app/telegram-messenger/id686449807?platform=ipad".to_string()),
-Url::from("https://apps.apple.com/us/app/fandom-videos-news-reviews/id1230063803".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/cold-call-by-harvard-business/id1156646189".to_string()),
-Url::from("https://vpp.itunes.apple.com".to_string()),
-Url::from("http://itunes.apple.com/us/app/chmplus-chm-reader/id441521818?mt=8".to_string()),
-Url::from("http://itunes.apple.com/cn/app/id441756221?mt=8".to_string()),
-Url::from("https://itunes.apple.com/app/snailgit-free-git-client-with-finder-integration/id1099475282?mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/snailgit/id847260112?l=en&mt=12".to_string()),
-Url::from("http://itunes.apple.com/app/ama-connect/id1355068050?uo=5&at=10l9yE".to_string()),
-Url::from("http://developer.apple.com/documentation/MacOSX/Conceptual/BPRuntimeConfig/index.html".to_string()),
-Url::from("https://support.apple.com/en-us/HT202074".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/pytorch-developer-podcast/id1566080008".to_string()),
-Url::from("https://itunes.apple.com/us/app/inaturalist/id421397028?mt=8".to_string()),
-Url::from("https://itunes.apple.com/app/apple-store/id736535961?pt=2334150&ct=Coursera%20Web%20Promo%20Banner&mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/privacy-management/id1365754887?mt=8".to_string()),
-Url::from("https://pcr.apple.com/id1264567358".to_string()),
-Url::from("https://support.apple.com/en-is/guide/iphone/iph4f4cbd242/14.0/ios/14.0".to_string()),
-Url::from("https://itunes.apple.com/us/app/onlyoffice-documents/id944896972".to_string()),
-Url::from("http://support.apple.com/kb/HT1677".to_string()),
-Url::from("https://apps.apple.com/app/impact-com/id1629721141".to_string()),
-Url::from("https://apps.apple.com/app/id1671420139".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/alchemy/id1503608708".to_string()),
-Url::from("https://apps.apple.com/us/app/haaretz-english-edition/id504537897".to_string()),
-Url::from("https://apps.apple.com/us/app/microsoft-whiteboard/id1352499399".to_string()),
-Url::from("https://support.apple.com/en-gb/guide/apple-business-connect/abcbb749656b/web".to_string()),
-Url::from("https://www.apple.com/business/connect".to_string()),
-Url::from("https://itunes.apple.com/us/app/id447919187?mt=8".to_string()),
-Url::from("https://apps.apple.com/us/app/id447919187?platform=ipad".to_string()),
-Url::from("https://itunes.apple.com/us/app/wireguard/id1451685025?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/wireguard/id1441195209?ls=1&mt=8".to_string()),
-Url::from("https://apps.apple.com/us/app/geekbench-ml/id1563487010".to_string()),
-Url::from("https://apps.apple.com/us/app/geekbench-6/id1565728895".to_string()),
-Url::from("https://www.apple.com/wss/fonts?families=SF+Pro,v1|SF+Pro+Icons,v1".to_string()),
-Url::from("https://www.apple.com/privacy".to_string()),
-Url::from("https://www.apple.com/news".to_string()),
-Url::from("https://apps.apple.com/us/app/id1460576753".to_string()),
-Url::from("https://apps.apple.com/fr/app/engie-gaz-et-electricit%C3%A9/id1104269908".to_string()),
-Url::from("https://apps.apple.com/us/app/bloomberg-business-news/id281941097".to_string()),
-Url::from("https://itunes.apple.com/us/app/glassdoor-job-search-salaries/id589698942?mt=8&l=eng".to_string()),
-Url::from("https://itunes.apple.com/app/id980888073".to_string()),
-Url::from("https://support.apple.com/en-gb/guide/safari/sfri11471/mac".to_string()),
-Url::from("https://support.apple.com/en-gb/HT201265".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/99-invisible/id394775318?mt=2".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/99-invisible/id394775318?at=10l9zE".to_string()),
-Url::from("https://tv.apple.com/vn/show/acapulco/umc.cmc.8xrkg9zywke7g6a9ahmvpr3l".to_string()),
-Url::from("https://www.apple.com/newsroom/2022/04/apple-unveils-the-best-photos-from-the-shot-on-iphone-macro-challenge".to_string()),
-Url::from("https://support.apple.com/guide/safari/customize-website-notifications-sfri40734/16.1/mac/13.0".to_string()),
-Url::from("http://www.apple.com/safari".to_string()),
-Url::from("https://apps.apple.com/app/id1622656471".to_string()),
-Url::from("https://itunes.apple.com/us/app/ip2location-geolocation/id771597553?ls=1&mt=8".to_string()),
-Url::from("https://apps.apple.com/es/app/diario-abc/id336860348".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=PAMKMDHYQRZYRVK1T2SN&pageLoadId=PBbn8FzEc7Zkrq58&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://apps.apple.com/us/app/nyt-audio/id1549293936".to_string()),
-Url::from("https://itunes.apple.com/si/app/mcobiss/id687682312?mt=8".to_string()),
-Url::from("https://support.apple.com/en-gb/accessibility".to_string()),
-Url::from("https://maps.apple.com/?q=48.8,2.3&t=k".to_string()),
-Url::from("https://maps.apple.com/?q=48.8,2.3&t=m".to_string()),
-Url::from("https://www.apple.com/legal/privacy/en-ww/cookies".to_string()),
-Url::from("https://www.apple.com/legal/privacy/en-ww".to_string()),
-Url::from("https://itunes.apple.com/ru/app/id1064581926".to_string()),
-Url::from("https://apps.apple.com/app/apple-store/id425194759?pt=417731&ct=enterprise_mobile_comms_LP&mt=8".to_string()),
-Url::from("https://apps.apple.com/app/apple-store/id1356686763?pt=417731&ct=lohp&mt=8".to_string()),
-Url::from("https://apps.apple.com/us/app/vimeo-create/id1491791513".to_string()),
-Url::from("https://itunes.apple.com/us/app/firefox-private-safe-browser/id989804926".to_string()),
-Url::from("https://support.apple.com/guide/security/controlling-app-access-to-files-secddd1d86a6/web".to_string()),
-Url::from("https://apps.apple.com/us/app/stc-training-club/id575468749".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/what-we-learned-today/id1465904347".to_string()),
-Url::from("https://itunes.apple.com/app/apple-store/id1036449551?pt=117888292&ct=discogs-app-page&mt=8".to_string()),
-Url::from("https://developer.apple.com/downloads".to_string()),
-Url::from("https://developer.apple.com/technologies/tools".to_string()),
-Url::from("https://developer.apple.com/macosx/launchd.html".to_string()),
-Url::from("https://feedbackassistant.apple.com".to_string()),
-Url::from("https://itunes.apple.com/us/app/xcode/id497799835".to_string()),
-Url::from("https://developer.apple.com/downloads/index.action".to_string()),
-Url::from("https://www.apple.com/itunes".to_string()),
-Url::from("https://support.apple.com/downloads#quicktime".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/software-engineering-institute-sei-podcast-series/id566573552?mt=2".to_string()),
-Url::from("https://itunes.apple.com/podcast/stephan-livera-podcast/id1415720320?mt=2".to_string()),
-Url::from("https://music.apple.com/za/post/1513061930".to_string()),
-Url::from("https://support.apple.com/kb/PH17191?locale=it_IT".to_string()),
-Url::from("https://apps.apple.com/us/app/the-wall-street-journal/id364387007?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/my-tune-device/id1100377074?ls=1&mt=8".to_string()),
-Url::from("https://apps.apple.com/ie/developer/the-irish-times-ltd/id365544597".to_string()),
-Url::from("https://itunes.apple.com/us/book/id1457507869".to_string()),
-Url::from("https://itunes.apple.com/us/app/cctvnews/id922456579".to_string()),
-Url::from("https://support.apple.com/kb/PH21411?locale=en_US".to_string()),
-Url::from("https://support.apple.com/kb/PH5042?locale=en_US".to_string()),
-Url::from("https://support.apple.com/guide/apple-configurator-2/revive-or-restore-a-mac-with-apple-silicon-apdd5f3c75ad/mac".to_string()),
-Url::from("https://itunes.apple.com/us/app/propublica/id355298887?mt=8".to_string()),
-Url::from("https://apps.apple.com/app/duckduckgo-privacy-browser/id663592361?ref=spreadprivacy.com".to_string()),
-Url::from("https://apps.apple.com/us/app/duckduckgo-privacy-browser/id663592361?ref=spreadprivacy.com".to_string()),
-Url::from("https://apps.apple.com/us/app/hubspot/id1107711722".to_string()),
-Url::from("https://support.apple.com/en-us/HT204915".to_string()),
-Url::from("https://support.apple.com/en-us/HT202303".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/effs-how-to-fix-the-internet/id1539719568".to_string()),
-Url::from("https://support.apple.com/en-us/HT201060".to_string()),
-Url::from("https://www.apple.com/newsroom/2022/12/apple-advances-user-security-with-powerful-new-data-protections".to_string()),
-Url::from("https://developer.apple.com/find-my".to_string()),
-Url::from("https://support.apple.com/en-us/HT201088".to_string()),
-Url::from("https://apps.apple.com/us/app/remote-desktop-mobile/id714464092".to_string()),
-Url::from("https://itunes.apple.com/de/podcast/filterblase-t3n-pioneers-podcast/id1115601393?mt=2".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/an-arm-and-a-leg/id1438778444".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/american-diagnosis-with-dr-celine-gounder/id1282044849?uo=4".to_string()),
-Url::from("https://podcasts.apple.com/podcast/where-it-hurts/id1529977695?mt=2".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/what-the-health/id1253607372?mt=2".to_string()),
-Url::from("https://apps.apple.com/jp/app/amazon-seller/id794141485".to_string()),
-Url::from("https://developer.apple.com/design/tips".to_string()),
-Url::from("https://developer.apple.com/library/prerelease/ios//documentation/AVFoundation/Reference/AVCaptureSession_Class/index.html".to_string()),
-Url::from("https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/DocumentPickerProgrammingGuide/AccessingDocuments/AccessingDocuments.html".to_string()),
-Url::from("https://developer.apple.com/videos/wwdc/2012".to_string()),
-Url::from("https://developer.apple.com/videos/play/wwdc2016/413".to_string()),
-Url::from("https://developer.apple.com/documentation/appkit/nsstandardkeybindingresponding?language=objc".to_string()),
-Url::from("https://developer.apple.com/documentation/foundation/nsdata/3174960-compressed".to_string()),
-Url::from("https://support.apple.com/en-gb/guide/preview/prvw587dd90f/mac".to_string()),
-Url::from("https://developer.apple.com/reference/xctest".to_string()),
-Url::from("https://developer.apple.com/documentation/mapkit/mkgeojsondecoder".to_string()),
-Url::from("https://support.apple.com/en-gb/guide/mac-help/mchlp1560/mac".to_string()),
-Url::from("https://developer.apple.com/sf-symbols".to_string()),
-Url::from("https://support.apple.com/en-gb/guide/mac-help/mchlp1046/mac".to_string()),
-Url::from("https://developer.apple.com/documentation/appkit/nsview/1483440-bitmapimagerepforcachingdisplay".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/IDEs/Conceptual/AppDistributionGuide/MaintainingProfiles/MaintainingProfiles.html".to_string()),
-Url::from("https://developer.apple.com/library/ios/qa/qa1633/_index.html".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/LanguagesUtilities/Conceptual/iTunesConnect_Guide/Chapters/CreatingiTunesConnectRecord.html".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/userexperience/Conceptual/MobileHIG/index.html#//apple_ref/doc/uid/TP40006556".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/IDEs/Conceptual/AppDistributionGuide/TestingYouriOSApp/TestingYouriOSApp.html#//apple_ref/doc/uid/TP40012582-CH8-SW1".to_string()),
-Url::from("https://developer.apple.com/search".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/StringsTextFonts/Conceptual/TextAndWebiPhoneOS/UsingTextClasses/UsingTextClasses.html".to_string()),
-Url::from("https://developer.apple.com/library/archive/technotes/tn2339/_index.html".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/UserExperience/Conceptual/MobileHIG/IconMatrix.html".to_string()),
-Url::from("https://developer.apple.com/documentation/imageio/3333271-cganimateimageaturlwithblock".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uidocument/1619945-statechangednotification".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/UIKit/Reference/UIActivityViewController_Class".to_string()),
-Url::from("https://developer.apple.com/documentation/swift_packages/supportedplatform/3788290-maccatalyst".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/view_controllers/preserving_your_app_s_ui_across_launches".to_string()),
-Url::from("https://books.apple.com/us/book/id1551005489".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/StringsTextFonts/Conceptual/TextAndWebiPhoneOS/CustomTextProcessing/CustomTextProcessing.html".to_string()),
-Url::from("https://developer.apple.com/download/more".to_string()),
-Url::from("https://apps.apple.com/gb/app/scratch/id1671420139".to_string()),
-Url::from("https://developer.apple.com/documentation/security/password_autofill/customizing_password_autofill_rules".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/general/conceptual/CocoaEncyclopedia/Introduction/Introduction.html#//apple_ref/doc/uid/TP40010810-CH1-SW1".to_string()),
-Url::from("https://itunes.apple.com/gb/course/developing-ios-7-apps-for/id733644550".to_string()),
-Url::from("https://developer.apple.com/library/ios/samplecode/PhotoScroller/Introduction/Intro.html".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/general/conceptual/CocoaEncyclopedia/Initialization/Initialization.html#//apple_ref/doc/uid/TP40010810-CH6-SW3".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uicollectionviewcompositionallayout".to_string()),
-Url::from("https://developer.apple.com/videos/play/wwdc2015/219".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/General/Conceptual/DevPedia-CocoaCore/AppID.html".to_string()),
-Url::from("https://support.apple.com/en-bw/guide/activity-monitor/actmaea30277/mac".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/UserExperience/Conceptual/AutolayoutPG/VisualFormatLanguage.html".to_string()),
-Url::from("https://apps.apple.com/app/id1558358855".to_string()),
-Url::from("https://apps.apple.com/us/app/rested-simple-http-requests/id421879749".to_string()),
-Url::from("https://developer.apple.com/documentation/swiftui/environmentvalues".to_string()),
-Url::from("https://developer.apple.com/swift/blog/?id=25".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html#//apple_ref/doc/uid/TP40009249-SW10".to_string()),
-Url::from("https://developer.apple.com/documentation/swiftui/state_and_data_flow".to_string()),
-Url::from("https://support.apple.com/en-gb/HT206481#thirdparty".to_string()),
-Url::from("https://developer.apple.com/library/ios/technotes/tn2259/_index.html".to_string()),
-Url::from("https://developer.apple.com/videos/wwdc/2014".to_string()),
-Url::from("https://testflight.apple.com".to_string()),
-Url::from("https://developer.apple.com/documentation/Charts".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/UserExperience/Conceptual/AutolayoutPG/AdoptingAutoLayout/AdoptingAutoLayout.html".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/Cocoa/Conceptual/ProgrammingWithObjectiveC/WorkingwithBlocks/WorkingwithBlocks.html".to_string()),
-Url::from("https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/CodingGuidelines/Articles/NamingMethods.html#//apple_ref/doc/uid/20001282-BCIGIJJF".to_string()),
-Url::from("https://apps.apple.com/app/id1578965002".to_string()),
-Url::from("https://developer.apple.com/documentation/charts/linemark".to_string()),
-Url::from("https://apps.apple.com/app/id1558358855?action=write-review".to_string()),
-Url::from("https://developer.apple.com/library/ios/qa/qa1713/_index.html".to_string()),
-Url::from("https://www.apple.com/legal/internet-services/itunes/testflight/sren/terms.html".to_string()),
-Url::from("https://developer.apple.com/library/mac/releasenotes/ObjectiveC/RN-TransitioningToARC/Introduction/Introduction.html".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/AVFoundation/Reference/AVCaptureVideoPreviewLayer_Class/index.html".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/uncast/id1566634831".to_string()),
-Url::from("https://apps.apple.com/us/app/fliff-sports-picks/id1489145500?see-all=reviews".to_string()),
-Url::from("https://apps.apple.com/us/app/fliff/id1489145500?ls=1".to_string()),
-Url::from("https://support.apple.com/en-gb/HT205280".to_string()),
-Url::from("https://developer.apple.com/reference/appkit/nsaccessibility".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/between-the-brackets-a-mediawiki-podcast/id1346062023?mt=2".to_string()),
-Url::from("https://support.apple.com/guide/safari/manage-cookies-sfri11471/mac".to_string()),
-Url::from("https://developer.apple.com/videos/play/wwdc2023/10121".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/the-stack-overflow-podcast/id1033688462".to_string()),
-Url::from("https://apps.apple.com/au/app/amazon-seller/id794141485".to_string()),
-Url::from("https://www.apple.com/newsroom/2023/05/apple-launches-20-fun-new-games-for-its-award-winning-apple-arcade-service".to_string()),
-Url::from("https://apps.apple.com/us/app/brave-private-web-browser/id1052879175?mt=8&ign-mpt=uo%3D4".to_string()),
-Url::from("https://support.apple.com/en-il/guide/safari/manage-cookies-and-website-data-sfri11471/mac".to_string()),
-Url::from("https://apps.apple.com/app/google-chrome".to_string()),
-Url::from("https://developer.apple.com/documentation/hypervisor".to_string()),
-Url::from("https://support.apple.com/guide/safari/manage-cookies-and-website-data-sfri11471/14.0/mac/11.0".to_string()),
-Url::from("https://developer.apple.com/reference/foundation/nsurl/1417051-startaccessingsecurityscopedreso?language=objc".to_string()),
-Url::from("https://developer.apple.com/documentation/coreservices/1441725-lscopydefaulthandlerforurlscheme?language=objc".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/Foundation/Reference/NSUserActivity_Class/index.html#//apple_ref/occ/instp/NSUserActivity/activityType".to_string()),
-Url::from("https://developer.apple.com/documentation/foundation/nsusernotification".to_string()),
-Url::from("https://developer.apple.com/design/human-interface-guidelines/notifications".to_string()),
-Url::from("https://developer.apple.com/documentation/appkit/nsaboutpaneloptioncredits?language=objc".to_string()),
-Url::from("https://developer.apple.com/library/content/documentation/General/Reference/InfoPlistKeyReference/Articles/CocoaKeys.html#//apple_ref/doc/uid/TP40009251-SW35".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/UserExperience/Conceptual/Handoff/HandoffFundamentals/HandoffFundamentals.html".to_string()),
-Url::from("https://itunes.apple.com/gb/app/fudget-budget-planner-personal-finance-tracker/id1211707701?mt=12".to_string()),
-Url::from("https://developer.apple.com/library/content/documentation/Security/Conceptual/AppSandboxDesignGuide/AppSandboxInDepth/AppSandboxInDepth.html#//apple_ref/doc/uid/TP40011183-CH3-SW16".to_string()),
-Url::from("https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/EventOverview/EventArchitecture/EventArchitecture.html#//apple_ref/doc/uid/10000060i-CH3-SW7".to_string()),
-Url::from("https://developer.apple.com/documentation/appkit/nsapplication/1428608-servicesmenu?language=objc".to_string()),
-Url::from("https://developer.apple.com/documentation/usernotifications/unnotificationresponse".to_string()),
-Url::from("https://developer.apple.com/library/archive/technotes/tn2150/_index.html".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html#//apple_ref/doc/uid/TP40009249-102207-TPXREF115".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/general/Reference/InfoPlistKeyReference/Articles/AboutInformationPropertyListFiles.html".to_string()),
-Url::from("https://developer.apple.com/reference/appkit/nsapplication/1428479-orderfrontstandardaboutpanelwith?language=objc".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/you-are-not-so-smart-attention/id521594713?i=113727803".to_string()),
-Url::from("https://apps.apple.com/us/app/bloomberg-b-unit/id1344778873".to_string()),
-Url::from("https://apps.apple.com/app/flat/id1598891661".to_string()),
-Url::from("https://support.apple.com/es-es/HT201265".to_string()),
-Url::from("https://support.apple.com/kb/PH19214?viewlocale=es_ES&locale=es_ES".to_string()),
-Url::from("https://apps.apple.com/us/app/geo-measure-area-calculator/id1099413961".to_string()),
-Url::from("https://apps.apple.com/us/app/kml-viewer-and-converter/id1457028727".to_string()),
-Url::from("https://developer.apple.com/documentation/safari-release-notes/safari-16_1-release-notes".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/css-tricks-screencasts/id273881728".to_string()),
-Url::from("https://developer.apple.com/passkeys".to_string()),
-Url::from("https://www.apple.com/accessibility/vision".to_string()),
-Url::from("https://podcasts.apple.com/th/podcast/143-fighting-the-climate-crisis-with-data/id1336150688?i=1000618315641".to_string()),
-Url::from("https://apps.apple.com/us/app/permanent-archive/id1571883070".to_string()),
-Url::from("https://support.apple.com/en-gb/guide/safari/sfri35610/mac".to_string()),
-Url::from("https://support.apple.com/HT204416".to_string()),
-Url::from("https://itunes.apple.com/fr/app/glassdoor-job-search-salaries/id589698942?mt=8&l=fra".to_string()),
-Url::from("https://itunes.apple.com/nl/app/glassdoor-job-search-salaries/id589698942?mt=8&l=nld".to_string()),
-Url::from("https://apps.apple.com/cz/app/id766443283?l=cs&mt=8".to_string()),
-Url::from("https://support.apple.com/en-euro/guide/safari/sfri11471/mac".to_string()),
-Url::from("https://itunes.apple.com/us/app/cdl-test-genie/id1447415467?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/dmv-genius-permit-practice/id513850893?mt=8".to_string()),
-Url::from("https://itunes.apple.com/app/apple-store/id1145737091".to_string()),
-Url::from("https://itunes.apple.com/us/app/famous-birthdays-celebrity/id646707938".to_string()),
-Url::from("https://itunes.apple.com/us/app/disconnect-privacy-pro-entire/id1057771839?ls=1&mt=8".to_string()),
-Url::from("https://www.apple.com/macbook-air-13-and-15-m2".to_string()),
-Url::from("https://www.apple.com/mac-pro".to_string()),
-Url::from("https://support.apple.com/en-us/HT210748".to_string()),
-Url::from("https://www.apple.com/mac-studio".to_string()),
-Url::from("https://support.apple.com/guide/mac-help/calibrate-your-display-mchlp1109/mac".to_string()),
-Url::from("https://developer.apple.com/documentation/swiftdata".to_string()),
-Url::from("https://apps.apple.com/us/app/uptime-robot-app/id1104878581?utm_source=homepage-section&utm_campaign=website-app-promo&ct=homepage-section-below".to_string()),
-Url::from("https://apps.apple.com/us/app/uptime-robot-app/id1104878581?utm_source=homepage-section&utm_campaign=website-app-promo&ct=homepage-section".to_string()),
-Url::from("https://itunes.apple.com/us/app/lmax-digital-trading/id1375900638".to_string()),
-Url::from("http://www.apple.com/fr/safari".to_string()),
-Url::from("https://support.apple.com/guide/safari/manage-cookies-and-website-data-sfri11471".to_string()),
-Url::from("https://support.apple.com/da-dk/HT201265".to_string()),
-Url::from("https://support.apple.com/da-dk/guide/safari/sfri11471/mac".to_string()),
-Url::from("https://developer.apple.com/documentation/quartzcore".to_string()),
-Url::from("https://apps.apple.com/mx/app/scotiam%C3%B3vil-mx/id720812080".to_string()),
-Url::from("https://developer.apple.com/documentation/virtualization/running_intel_binaries_in_linux_vms_with_rosetta".to_string()),
-Url::from("https://apps.apple.com/app/sketch-view-and-mirror/id1609224699".to_string()),
-Url::from("http://itunes.apple.com/us/app/faonow/id877618174?ls=1&mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/phys-org/id356443503?mt=8".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/programming-love-with-martin-kleppmann/id1518407590?i=1000495317576".to_string()),
-Url::from("https://podcasts.apple.com/ph/podcast/designing-data-intensive-applications-with-martin/id1531450276?i=1000533284011".to_string()),
-Url::from("https://apps.apple.com/app/datadog/id1391380318".to_string()),
-Url::from("https://help.apple.com/itc/podcasts_connect/#/itcbaf351599".to_string()),
-Url::from("https://help.apple.com/itc/podcasts_connect/#/itc2b3780e76".to_string()),
-Url::from("https://help.apple.com/itc/podcasts_connect/#/itcb54353390".to_string()),
-Url::from("https://support.apple.com/en-us/HT201762".to_string()),
-Url::from("https://podcastsconnect.apple.com".to_string()),
-Url::from("https://appleid.apple.com/account#!&page=create".to_string()),
-Url::from("https://apps.apple.com/app/time.is/id857189816".to_string()),
-Url::from("https://podcasts.apple.com/bg/podcast/the-recursive-podcast/id1576538398".to_string()),
-Url::from("https://apps.apple.com/jp/app/sail-japanese-conversations/id1434041945".to_string()),
-Url::from("https://www.apple.com/newsroom/2020/05/apple-to-host-virtual-worldwide-developers-conference-beginning-june-22".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/true-stories-of-good-people/id1389955443".to_string()),
-Url::from("https://support.apple.com/kb/PH21411?locale=nl_NL&viewlocale=en_US".to_string()),
-Url::from("https://support.apple.com/kb/PH21411?locale=nl_NL&viewlocale=fr_FR".to_string()),
-Url::from("https://itunes.apple.com/us/app/shadertoy/id717961814".to_string()),
-Url::from("https://itunes.apple.com/it/app/la-repubblica-mobile/id313084825?mt=8".to_string()),
-Url::from("https://apps.apple.com/us/developer/wolfram-group-llc/id715130979".to_string()),
-Url::from("https://apps.apple.com/us/app/wolfram-alpha/id548861535".to_string()),
-Url::from("https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/PrioritizeWorkAtTheTaskLevel.html".to_string()),
-Url::from("https://itunes.apple.com/us/app/jewish-virtual-library/id1336516435?mt=8".to_string()),
-Url::from("https://apps.apple.com/ca/app/snikket/id1545164189".to_string()),
-Url::from("https://developer.apple.com/wwdc".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/what-the-health/id1253607372?mt=2".to_string()),
-Url::from("https://apps.apple.com/us/app/uber/id368677368?uclick_id=c33a8d2c-1390-4ad3-9c63-bb9b1997e716".to_string()),
-Url::from("https://itunes.apple.com/us/app/privacy-monitor/id1450531857?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/privacy-monitor/id1451670029?mt=12".to_string()),
-Url::from("https://support.apple.com/en-us/HT205223".to_string()),
-Url::from("https://opensource.apple.com/source/xnu/xnu-7195.81.3/EXTERNAL_HEADERS/mach-o/fixup-chains.h.auto.html".to_string()),
-Url::from("https://itunes.apple.com/us/app/kvikmyndir.is/id1191288272?ls=1&mt=8".to_string()),
-Url::from("https://support.apple.com/en-us/HT202663".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/cold-call/id1156646189".to_string()),
-Url::from("https://www.apple.com/de/macos".to_string()),
-Url::from("https://developer.apple.com/opensource".to_string()),
-Url::from("https://www.apple.com/newsroom/2023/06/ios-17-makes-iphone-more-personal-and-intuitive".to_string()),
-Url::from("https://www.apple.com/newsroom/2023/06/tvos-17-brings-facetime-and-video-conferencing-to-apple-tv-4k".to_string()),
-Url::from("https://www.apple.com/newsroom/2023/06/apple-introduces-the-15-inch-macbook-air".to_string()),
-Url::from("https://www.apple.com/newsroom/2023/06/introducing-watchos-10-a-milestone-update-for-apple-watch".to_string()),
-Url::from("https://www.apple.com/newsroom/2023/06/ipados-17-brings-new-levels-of-personalization-and-versatility-to-ipad".to_string()),
-Url::from("https://www.apple.com/newsroom/2023/06/apple-introduces-m2-ultra".to_string()),
-Url::from("https://www.apple.com/newsroom/2023/06/macos-sonoma-brings-new-capabilities-for-elevating-productivity-and-creativity".to_string()),
-Url::from("https://support.apple.com/en-us/HT212850".to_string()),
-Url::from("https://developer.apple.com/homekit".to_string()),
-Url::from("https://www.apple.com/newsroom/2023/06/apple-announces-powerful-new-privacy-and-security-features".to_string()),
-Url::from("https://developer.apple.com/videos/play/wwdc2023/10120".to_string()),
-Url::from("https://apps.apple.com/app/lmax-global-trading/id884042608".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/damn-interesting/id571835813".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/damn-interesting/id571835813?mt=2".to_string()),
-Url::from("https://itunes.apple.com/ro/app/iaBilet/id899564929".to_string()),
-Url::from("https://apps.apple.com/app/id6444779277".to_string()),
-Url::from("https://podcasts.apple.com/dk/podcast/datatilsynet-podcast-bliv-klogere-p%C3%A5-gdpr/id1481257016".to_string()),
-Url::from("https://itunes.apple.com/pl/app/podr%C4%99czny-s%C5%82ownik-hiszpa%C5%84sko-polski-pwn/id1120160080?mt=8&utm_source=ref&utm_medium=web&utm_campaign=6P_0916_ref_201609".to_string()),
-Url::from("https://tv.apple.com/movie/evil-dead-rise/umc.cmc.1qv727yx0ev1kdt33soi1xbmy?uo=5&itsct=tv_box_link&itscg=30200&at=1010l369s".to_string()),
-Url::from("https://tv.apple.com/movie/spider-man-into-the-spider-verse/umc.cmc.4a2ypxddplsdsahwxrc5fkz4x?uo=5&itsct=tv_box_link&itscg=30200&at=1010l369s".to_string()),
-Url::from("https://apps.apple.com/app/id980888073#?platform=iphone".to_string()),
-Url::from("https://apps.apple.com/app/id980888073#?platform=mac".to_string()),
-Url::from("https://apps.apple.com/gb/app/microsoft-office/id541164041".to_string()),
-Url::from("https://apps.apple.com/gb/app/microsoft-teams/id1113153706".to_string()),
-Url::from("https://apps.apple.com/us/app/skype-for-iphone/id304878510".to_string()),
-Url::from("https://apps.apple.com/us/app/shopify-inbox/id1301681854".to_string()),
-Url::from("https://apps.apple.com/us/app/facebook-ads-manager/id964397083".to_string()),
-Url::from("https://apps.apple.com/us/app/wix-owner-website-builder/id1545924344".to_string()),
-Url::from("https://itunes.apple.com/us/app/xcode/id497799835?mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/wix-com/id1099748482".to_string()),
-Url::from("https://apps.apple.com/us/developer/wix-com-inc/id407141669".to_string()),
-Url::from("https://apps.apple.com/gb/app/xbox-game-pass/id1374542474".to_string()),
-Url::from("https://apps.apple.com/us/app/oculus/id1366478176".to_string()),
-Url::from("https://apps.apple.com/ca/app/shop-package-order-tracker/id1223471316".to_string()),
-Url::from("https://apps.apple.com/us/app/microsoft-outlook/id951937596".to_string()),
-Url::from("https://apps.apple.com/app/facebook/id284882215".to_string()),
-Url::from("https://apps.apple.com/us/app/shopify-point-of-sale-pos/id686830644".to_string()),
-Url::from("https://apps.apple.com/us/app/dine-by-wix/id1561524113".to_string()),
-Url::from("https://apps.apple.com/us/app/shopify-your-ecommerce-store/id371294472".to_string()),
-Url::from("https://support.apple.com/fr-afri/guide/safari/sfri11471/mac".to_string()),
-Url::from("https://itunes.apple.com/us/app/swann-auction-galleries/id1440439183?ls=1&mt=8".to_string()),
-Url::from("http://support.apple.com/kb/DL1251".to_string()),
-Url::from("https://support.apple.com/kb/DL1283".to_string()),
-Url::from("http://opensource.apple.com/source/IOUSBFamily/IOUSBFamily-458.4.1/AppleUSBOHCI/Classes/AppleUSBOHCI_UIM.cpp".to_string()),
-Url::from("https://discussions.apple.com/thread/123054?start=0&tstart=0".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/the-photo-detective/id1255965884?mt=2&mc_cid=67037096ee&mc_eid=[UNIQID]".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/the-photo-detective/id1255965884?mt=2".to_string()),
-Url::from("https://itunes.apple.com/us/app/audubon-bird-guide-north-america/id333227386?mt=8&utm_source=bird-guide&utm_medium=national&utm_campaign=iosapp".to_string()),
-Url::from("https://apps.apple.com/us/app/clips-clipboard-manager/id1549350243".to_string()),
-Url::from("https://store.apple.com/xc/product/MJYT2AM/A".to_string()),
-Url::from("https://apps.apple.com/us/app/obscura-3-pro-camera/id1579306989".to_string()),
-Url::from("https://apps.apple.com/us/app/halide-mark-ii-pro-camera/id885697368".to_string()),
-Url::from("https://developer.apple.com/documentation/xcode-release-notes".to_string()),
-Url::from("http://developer.apple.com".to_string()),
-Url::from("http://apple.com".to_string()),
-Url::from("https://developer.apple.com/library/mac/#documentation/Cocoa/Conceptual/MemoryMgmt/Articles/mmRules.html".to_string()),
-Url::from("https://apps.apple.com/us/app/keenetic/id1413650865".to_string()),
-Url::from("http://support.apple.com/kb/PH5067".to_string()),
-Url::from("https://www.apple.com/privacy/privacy-policy".to_string()),
-Url::from("http://developer.apple.com/documentation/Carbon/Conceptual/CoreText_Programming/Overview/chapter_2_section_4.html#//apple_ref/doc/uid/TP40005533-CH3-DontLinkElementID_10".to_string()),
-Url::from("http://www.apple.com/macosx/features/bootcamp.html".to_string()),
-Url::from("https://apps.apple.com/it/app/vanity-fair-italia/id539933331".to_string()),
-Url::from("https://support.apple.com/kb/PH19214".to_string()),
-Url::from("https://support.apple.com/en-bh/HT209143".to_string()),
-Url::from("https://itunes.apple.com/us/app/appchoices/id894822870".to_string()),
-Url::from("https://support.apple.com/safari".to_string()),
-Url::from("https://support.apple.com/es-es/safari".to_string()),
-Url::from("https://podcasts.apple.com/il/podcast/cpradio/id1491720376".to_string()),
-Url::from("https://apps.apple.com/ca/app/thestar-com-iphone/id379481068".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/78-logflare-with-chase-granberry/id1516100616?i=1000545628124".to_string()),
-Url::from("https://support.apple.com/en-gb/guide/safari/manage-cookies-and-website-data-sfri11471/mac".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/73-elixir-at-supabase-with-paul-copplestone/id1516100616?i=1000542087899".to_string()),
-Url::from("https://developer.apple.com/library/mac/documentation/MobileCoreServices/Reference/UTTypeRef/#//apple_ref/doc/constant_group/UTI_Text_Types".to_string()),
-Url::from("https://developer.apple.com/library/prerelease/ios/documentation/Foundation/Reference/NSItemProvider_Class/index.html".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/Foundation/Reference/NSExtensionItem_Class/#//apple_ref/occ/instp/NSExtensionItem/attachments".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/Foundation/Reference/NSExtensionItem_Class".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/Foundation/Reference/NSExtensionContext_Class".to_string()),
-Url::from("https://developer.apple.com/library/prerelease/ios/documentation/General/Conceptual/ExtensibilityPG/NotificationCenter.html".to_string()),
-Url::from("https://developer.apple.com/videos/play/wwdc2022/110362".to_string()),
-Url::from("https://music.apple.com/artist/249646".to_string()),
-Url::from("https://developer.apple.com/library/content/documentation/DeveloperTools/Conceptual/MachOTopics/1-Articles/x86_64_code.html".to_string()),
-Url::from("https://opensource.apple.com/source/ld64/ld64-123.2.1/src/abstraction/MachOTrie.hpp".to_string()),
-Url::from("https://support.apple.com/en-in/guide/safari/sfri11471/mac".to_string()),
-Url::from("https://apps.apple.com/app/tmbapp-metro-bus-barcelona/id387847254?pt=216671&ct=web_tmb&mt=8".to_string()),
-Url::from("https://itunes.apple.com/it/app/itpc-carabinieri/id858588594?mt=8".to_string()),
-Url::from("https://itunes.apple.com/tw/app/dramexchange-world-leading/id1080865522?mt=8".to_string()),
-Url::from("http://www.apple.com/downloads/macosx/development_tools/svnx.html".to_string()),
-Url::from("https://apps.apple.com/ch/app/hostpoint-meet/id1511503042".to_string()),
-Url::from("http://www.apple.com/music/store".to_string()),
-Url::from("http://support.apple.com/kb/HT4009".to_string()),
-Url::from("http://www.apple.com/support/itunes/account".to_string()),
-Url::from("http://www.apple.com/support/ipad".to_string()),
-Url::from("https://podcasts.apple.com/podcast/id1559261829".to_string()),
-Url::from("https://developer.apple.com/library/mac/releasenotes/DeveloperTools/RN-Xcode/Chapters/xc6_release_notes.html#//apple_ref/doc/uid/TP40001051-CH4-SW10".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/unstyled/id1171140955".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/the-cs-ed-podcast/id1486344352".to_string()),
-Url::from("https://apps.apple.com/us/app/speechify-text-to-speech-audio/id1209815023".to_string()),
-Url::from("https://itunes.apple.com/es/app/marina-dor/id925265167?mt=8".to_string()),
-Url::from("https://www.apple.com/newsroom/2020/11/apple-unleashes-m1".to_string()),
-Url::from("https://developer.apple.com/news/?id=09032019a".to_string()),
-Url::from("https://developer.apple.com/library/archive/documentation/General/Conceptual/ExtensibilityPG/Finder.html".to_string()),
-Url::from("https://developer.apple.com/devcenter/download.action?path=/videos/wwdc_2012__hd/session_241__cocoa_interprocess_communication_with_xpc.mov".to_string()),
-Url::from("https://opensource.apple.com/source/text_cmds/text_cmds-68/wc/wc.c.auto.html".to_string()),
-Url::from("https://support.apple.com/guide/shortcuts-mac/welcome/mac".to_string()),
-Url::from("https://developer.apple.com/metal/shader-converter".to_string()),
-Url::from("https://support.apple.com/en-ie/HT202074".to_string()),
-Url::from("https://apps.apple.com/app/userscripts/id1463298887".to_string()),
-Url::from("http://linkmaker.itunes.apple.com".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/smart-author/id1298180300?mt=2".to_string()),
-Url::from("https://developer.apple.com/documentation/macos-release-notes/macos-13_4-release-notes".to_string()),
-Url::from("https://books.apple.com/book/packaging-for-apple-administrators/id1173928620?mt=11&app=itunes&at=1001lpyX&ct=newsletter".to_string()),
-Url::from("https://developer.apple.com/documentation/visionos".to_string()),
-Url::from("https://support.apple.com/en-us/HT202491".to_string()),
-Url::from("https://geo.itunes.apple.com/us/book/packaging-for-apple-administrators/id1173928620?ls=1&mt=11&at=1001lpyX".to_string()),
-Url::from("https://www.apple.com/newsroom/2023/05/app-store-stopped-more-than-2-billion-in-fraudulent-transactions-in-2022".to_string()),
-Url::from("https://developer.apple.com/wwdc23/10165".to_string()),
-Url::from("https://www.apple.com/newsroom/2023/05/apple-previews-live-speech-personal-voice-and-more-new-accessibility-features".to_string()),
-Url::from("https://support.apple.com/en-us/HT213544".to_string()),
-Url::from("https://developer.apple.com/documentation/Combine".to_string()),
-Url::from("https://developer.apple.com/documentation/foundation/userdefaults".to_string()),
-Url::from("https://support.apple.com/kb/DL1998".to_string()),
-Url::from("https://developer.apple.com/wwdc20/10185".to_string()),
-Url::from("https://books.apple.com/book/packaging-for-apple-administrators/id1173928620?mt=11&app=itunes&at=1001lpyX".to_string()),
-Url::from("https://developer.apple.com/wwdc21/10095".to_string()),
-Url::from("https://developer.apple.com/go/?id=macos-10.15-sdk-rn".to_string()),
-Url::from("https://developer.apple.com/documentation/safari-release-notes/safari-16_5-release-notes".to_string()),
-Url::from("https://developer.apple.com/documentation/swift/commandline#topics".to_string()),
-Url::from("https://support.apple.com/en-us/HT207245".to_string()),
-Url::from("https://developer.apple.com/wwdc21/10261".to_string()),
-Url::from("https://developer.apple.com/documentation/security/notarizing_macos_software_before_distribution".to_string()),
-Url::from("https://support.apple.com/en-us/HT202998".to_string()),
-Url::from("https://developer.apple.com/documentation/security/notarizing_macos_software_before_distribution/customizing_the_notarization_workflow".to_string()),
-Url::from("https://support.apple.com/en-us/HT213327".to_string()),
-Url::from("https://developer.apple.com/wwdc23/10053".to_string()),
-Url::from("https://support.apple.com/en-us/HT213765".to_string()),
-Url::from("https://support.apple.com/guide/certifications/welcome/web".to_string()),
-Url::from("https://support.apple.com/en-us/HT208891".to_string()),
-Url::from("https://support.apple.com/en-us/HT207243".to_string()),
-Url::from("https://www.apple.com/ios/ios-17-preview".to_string()),
-Url::from("https://developer.apple.com/wwdc23/10002".to_string()),
-Url::from("https://geo.itunes.apple.com/us/book/learning-the-bash-shell/id396890728?mt=11&at=1001lpyX".to_string()),
-Url::from("https://support.apple.com/en-us/HT213760".to_string()),
-Url::from("https://developer.apple.com/wwdc23/10263".to_string()),
-Url::from("https://developer.apple.com/documentation/SwiftUI".to_string()),
-Url::from("https://support.apple.com/en-us/HT213250".to_string()),
-Url::from("https://support.apple.com/guide/certifications/document-revision-history-apc353b1b736/web".to_string()),
-Url::from("https://developer.apple.com/videos/play/wwdc2021/10132".to_string()),
-Url::from("https://support.apple.com/en-us/HT207242".to_string()),
-Url::from("https://security.apple.com/blog/what-if-we-had-sockpuppet-in-ios16".to_string()),
-Url::from("https://support.apple.com/en-us/HT207936".to_string()),
-Url::from("https://support.apple.com/guide/deployment/document-revision-history-dep9d814c555/web".to_string()),
-Url::from("https://support.apple.com/en-us/HT213810".to_string()),
-Url::from("https://www.apple.com/ipados/ipados-17-preview".to_string()),
-Url::from("https://developer.apple.com/wwdc23/10039".to_string()),
-Url::from("https://support.apple.com/en-us/HT213812".to_string()),
-Url::from("https://support.apple.com/en-us/HT213757".to_string()),
-Url::from("https://support.apple.com/en-us/HT213407".to_string()),
-Url::from("https://developer.apple.com/design/human-interface-guidelines/designing-for-visionos".to_string()),
-Url::from("https://support.apple.com/en-us/HT207246".to_string()),
-Url::from("https://developer.apple.com/documentation/xcode/writing-symbol-documentation-in-your-source-files".to_string()),
-Url::from("https://developer.apple.com/wwdc23/10041".to_string()),
-Url::from("https://www.apple.com/newsroom/2023/05/apple-launches-new-concert-discovery-features-on-apple-maps-and-apple-music".to_string()),
-Url::from("https://developer.apple.com/wwdc23/10040".to_string()),
-Url::from("https://developer.apple.com/documentation/foundation/url".to_string()),
-Url::from("https://geo.itunes.apple.com/book/property-lists-preferences-profiles-for-apple-administrators/id1213903756?ls=1&mt=11&at=1001lpyX&ct=bookpage".to_string()),
-Url::from("https://developer.apple.com/wwdc21/10018".to_string()),
-Url::from("https://support.apple.com/en-us/HT207247".to_string()),
-Url::from("https://support.apple.com/en-us/HT213408".to_string()),
-Url::from("https://support.apple.com/en-us/HT213758".to_string()),
-Url::from("https://developer.apple.com/documentation/xcode-release-notes/xcode-15-release-notes".to_string()),
-Url::from("https://books.apple.com/book/terminal-and-shell/id1550617226?mt=11&app=itunes&at=1001lpyX".to_string()),
-Url::from("https://developer.apple.com/documentation/technotes/tn3147-migrating-to-the-latest-notarization-tool".to_string()),
-Url::from("https://support.apple.com/en-us/HT213761".to_string()),
-Url::from("https://developer.apple.com/wwdc21/10062".to_string()),
-Url::from("https://support.apple.com/en-us/HT213813".to_string()),
-Url::from("https://support.apple.com/en-us/HT213470".to_string()),
-Url::from("https://developer.apple.com/documentation/ios-ipados-release-notes/ios-ipados-16_5-release-notes".to_string()),
-Url::from("https://support.apple.com/en-us/HT204451".to_string()),
-Url::from("https://support.apple.com/guide/tv/whats-new-in-tvos-16-atvb457e13ae/tvos".to_string()),
-Url::from("https://www.apple.com/apple-vision-pro".to_string()),
-Url::from("https://developer.apple.com/tutorials/swiftui".to_string()),
-Url::from("https://support.apple.com/en-us/HT213268".to_string()),
-Url::from("https://support.apple.com/en-us/HT207244".to_string()),
-Url::from("https://developer.apple.com/documentation/swiftui/building-layouts-with-stack-views".to_string()),
-Url::from("https://support.apple.com/en-us/HT213317".to_string()),
-Url::from("https://developer.apple.com/documentation/corefoundation/preferences_utilities".to_string()),
-Url::from("https://www.apple.com/watchos/watchos-preview".to_string()),
-Url::from("https://developer.apple.com/wwdc23/10120".to_string()),
-Url::from("https://developer.apple.com/library/archive/technotes/tn2065/_index.html".to_string()),
-Url::from("https://support.apple.com/en-us/HT213417".to_string()),
-Url::from("https://developer.apple.com/wwdc20/10040".to_string()),
-Url::from("https://support.apple.com/en-us/HT213759".to_string()),
-Url::from("https://support.apple.com/en-us/HT213808".to_string()),
-Url::from("https://support.apple.com/en-us/HT213762".to_string()),
-Url::from("https://developer.apple.com/documentation/coreservices/launch_services".to_string()),
-Url::from("https://support.apple.com/en-us/HT202999".to_string()),
-Url::from("https://developer.apple.com/documentation/appkit/nsworkspace#topics".to_string()),
-Url::from("https://books.apple.com/book/moving-to-zsh/id1483591353?mt=11&app=itunes&at=1001lpyX".to_string()),
-Url::from("https://support.apple.com/guide/deployment/whats-new-dep950aed53e/web".to_string()),
-Url::from("https://support.apple.com/kb/HT213811".to_string()),
-Url::from("https://support.apple.com/en-us/HT210060".to_string()),
-Url::from("https://support.apple.com/en-us/HT208714".to_string()),
-Url::from("https://developer.apple.com/wwdc20/10119".to_string()),
-Url::from("https://developer.apple.com/documentation/tvos-release-notes/tvos-16_5-release-notes".to_string()),
-Url::from("https://developer.apple.com/documentation/swiftui/appstorage".to_string()),
-Url::from("https://support.apple.com/en-us/HT213764".to_string()),
-Url::from("https://developer.apple.com/bug-reporting".to_string()),
-Url::from("https://www.apple.com/macos/sonoma-preview".to_string()),
-Url::from("https://developer.apple.com/wwdc23/10254".to_string()),
-Url::from("https://books.apple.com/us/book/moving-to-zsh/id1483591353?mt=11&app=itunes&at=1001lpyX".to_string()),
-Url::from("https://developer.apple.com/documentation/xcode-release-notes/xcode-14_3_1-release-notes".to_string()),
-Url::from("https://support.apple.com/guide/personal-safety/welcome/web".to_string()),
-Url::from("https://support.apple.com/en-us/HT201222".to_string()),
-Url::from("https://support.apple.com/en-us/HT213752".to_string()),
-Url::from("https://geo.itunes.apple.com/us/book/bash-cookbook/id396898692?mt=11&at=1001lpyX".to_string()),
-Url::from("https://developer.apple.com/wwdc21/10192".to_string()),
-Url::from("https://support.apple.com/en-us/HT213436".to_string()),
-Url::from("https://developer.apple.com/learn/curriculum".to_string()),
-Url::from("https://developer.apple.com/wwdc23/102".to_string()),
-Url::from("https://geo.itunes.apple.com/nl/book/take-control-mac-command-line-terminal-second-edition/id1234016278?l=en&mt=11&at=1001lpyX".to_string()),
-Url::from("https://developer.apple.com/wwdc23/10256".to_string()),
-Url::from("https://www.apple.com/newsroom/2023/05/apple-brings-final-cut-pro-and-logic-pro-to-ipad".to_string()),
-Url::from("https://geo.itunes.apple.com/book/classic-shell-scripting/id396887958?mt=11&at=1001lpyX".to_string()),
-Url::from("https://support.apple.com/en-us/HT201224".to_string()),
-Url::from("https://geo.itunes.apple.com/us/book/packaging-for-apple-administrators/id1173928620?ls=1&mt=11&at=1001lpyX&ct=bookpage".to_string()),
-Url::from("https://support.apple.com/en-us/HT208050".to_string()),
-Url::from("https://books.apple.com/book/moving-to-zsh/id1483591353?mt=11&app=itunes&at=1001lpyX&ct=newsletter".to_string()),
-Url::from("https://support.apple.com/en-us/HT213809".to_string()),
-Url::from("https://developer.apple.com/wwdc23/101".to_string()),
-Url::from("https://developer.apple.com/visionos".to_string()),
-Url::from("https://support.apple.com/en-us/HT213814".to_string()),
-Url::from("https://developer.apple.com/documentation/appkit/nsworkspace".to_string()),
-Url::from("https://apps.apple.com/us/app/afends-au/id1568128486".to_string()),
-Url::from("https://podcasts.apple.com/ee/podcast/carbon-visibility-in-supply-chains/id1577110956?i=1000597928373".to_string()),
-Url::from("https://podcasts.apple.com/ee/podcast/the-what-and-why-of-sustainable-freight/id1577110956?i=1000595798531".to_string()),
-Url::from("https://podcasts.apple.com/ee/podcast/how-to-build-a-sustainable-freight-ecosystem/id1577110956?i=1000596983877".to_string()),
-Url::from("https://support.apple.com/kb/PH17191".to_string()),
-Url::from("https://support.apple.com/es_ES/downloads".to_string()),
-Url::from("https://www.apple.com/es/itunes".to_string()),
-Url::from("https://itunes.apple.com/us/app/raise-discounted-gift-cards/id922643195?mt=8".to_string()),
-Url::from("https://apps.apple.com/us/app/%E6%97%A5%E6%9C%AC%E5%A7%93%E6%B0%8F%E8%AA%9E%E6%BA%90%E8%BE%9E%E5%85%B8-%E3%82%AA%E3%83%95%E3%83%A9%E3%82%A4%E3%83%B3/id1471109324?mt=8".to_string()),
-Url::from("https://apps.apple.com/us/app/pocketbook-reader/id805488884".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/the-atlas-obscura-podcast/id1555769970".to_string()),
-Url::from("https://apps.apple.com/us/app/microsoft-teams/id1113153706".to_string()),
-Url::from("https://apps.apple.com/lt/app/seb-lietuva-m-bankas/id445885936".to_string()),
-Url::from("https://apps.apple.com/app/myfootprint-seb/id1589232172".to_string()),
-Url::from("https://support.apple.com/en-us/guide/safari/sfri11471/mac".to_string()),
-Url::from("https://itunes.apple.com/us/app/m1-finance/id1071915644".to_string()),
-Url::from("https://apps.apple.com/us/app/m1-finance/id1071915644".to_string()),
-Url::from("https://discussions.apple.com/thread/1107629".to_string()),
-Url::from("http://lists.apple.com/archives/Objc-language/2005/Aug/msg00021.html".to_string()),
-Url::from("http://lists.apple.com/archives/Objc-language/2005/Aug/threads.html#00019".to_string()),
-Url::from("http://developer.apple.com/TOOLS/Xcode".to_string()),
-Url::from("https://developer.apple.com/library/content/documentation/Xcode/Reference/xcode_ref-Asset_Catalog_Format/index.html".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/north-star-podcast/id1170914572".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/software-engineering-institute-sei-podcast-series/id566573552?mt=2".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/podcasting-smarter/id566573552?mt=2".to_string()),
-Url::from("https://apps.apple.com/us/app/3d%E3%82%B9%E3%82%AD%E3%83%A3%E3%83%8A/id1581786259".to_string()),
-Url::from("https://apps.apple.com/app/goodscan-3d/id1600905202".to_string()),
-Url::from("https://itunes.apple.com/us/app/voxel-draw/id1084634617".to_string()),
-Url::from("https://developer.apple.com/library/archive/documentation/TextFonts/Conceptual/CocoaTextArchitecture/TypoFeatures/TextSystemFeatures.html".to_string()),
-Url::from("https://itunes.apple.com/us/app/wikipedia/id324715238".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/graphistania-podcast-for-neo4j-graph-database-community/id975377379".to_string()),
-Url::from("https://support.apple.com/zh-cn/guide/iphone/iph10d7bdf26/ios".to_string()),
-Url::from("https://apps.apple.com/us/app/wave-invoicing/id881629660".to_string()),
-Url::from("https://support.apple.com/en-us/HT212608".to_string()),
-Url::from("https://apps.apple.com/us/app/tracker-mood-energy-diary/id1187822249#?platform=messages".to_string()),
-Url::from("https://support.apple.com/en-gb/guide/safari/welcome/mac".to_string()),
-Url::from("https://www.apple.com/mac-mini".to_string()),
-Url::from("https://www.apple.com/macbook-air".to_string()),
-Url::from("https://apps.apple.com/us/app/noisli/id862773459".to_string()),
-Url::from("http://support.apple.com/downloads".to_string()),
-Url::from("http://www.apple.com/support".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/lawyers-guns-money/id1332780055?mt=2".to_string()),
-Url::from("https://apps.apple.com/us/app/id1520190857".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/big-ideas-in-app-architecture/id1670998452".to_string()),
-Url::from("http://www.apple.com/fr/quicktime/download".to_string()),
-Url::from("https://geo.itunes.apple.com/us/app/inquire-by-tamper/id1045268097?mt=8&at=11lmME&ct=tamperio".to_string()),
-Url::from("http://www.apple.com/es/privacy/use-of-cookies".to_string()),
-Url::from("https://support.apple.com/en-us/HT205280".to_string()),
-Url::from("https://itunes.apple.com/app/volumio/id1268256519?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/infinity/id1136944945".to_string()),
-Url::from("https://itunes.apple.com/de/institution/universitat-konstanz/id604733564".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/cppcast/id968703120".to_string()),
-Url::from("https://itunes.apple.com/ie/app/houses-of-the-oireachtas/id783571697?mt=8".to_string()),
-Url::from("https://itunes.apple.com/app/payoneer/id663338402".to_string()),
-Url::from("https://apps.apple.com/us/app/mayo-clinic/id523220194".to_string()),
-Url::from("https://apps.apple.com/de/app/stuttgarter-zeitung-app/id517190273".to_string()),
-Url::from("https://apps.apple.com/de/app/stuttgarter-zeitung-de/id397131360".to_string()),
-Url::from("https://itunes.apple.com/app/depositphotos/id919098408".to_string()),
-Url::from("http://www.apple.com/macosx/lion".to_string()),
-Url::from("http://www.apple.com/ipad/#smart-cover".to_string()),
-Url::from("https://itunes.apple.com/us/app/seek-by-inaturalist/id1353224144".to_string()),
-Url::from("https://www.apple.com/maps".to_string()),
-Url::from("https://itunes.apple.com/app/apple-store/id982107779".to_string()),
-Url::from("https://developer.apple.com/app-sandboxing".to_string()),
-Url::from("https://developer.apple.com/library/mac/documentation/UserExperience/Conceptual/AppleHIGuidelines/IconsImages/IconsImages.html#//apple_ref/doc/uid/20000967-TP6".to_string()),
-Url::from("https://developer.apple.com/appstore/mac/resources/approval/guidelines.html".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/the-tim-ferriss-show/id863897795".to_string()),
-Url::from("https://apps.apple.com/us/app/jetpack-website-builder/id1565481562".to_string()),
-Url::from("https://apps.apple.com/ru/app/fandom-videos-news-reviews/id1230063803".to_string()),
-Url::from("https://apps.apple.com/us/app/refinitiv-workspace/id1481442629".to_string()),
-Url::from("https://itunes.apple.com/app/aliexpress-for-ipad/id854356557".to_string()),
-Url::from("https://itunes.apple.com/us/app/aliexpress/id436672029".to_string()),
-Url::from("https://apps.apple.com/jp/app/medaca-%E8%87%AA%E5%88%86%E3%81%AE%E5%81%A5%E5%BA%B7%E3%82%92%E5%8F%8E%E7%B4%8D%E3%81%99%E3%82%8B%E3%82%A2%E3%83%97%E3%83%AA/id1190040349?l=en".to_string()),
-Url::from("https://machinelearning.apple.com/updates/apple-scholars-aiml-2022".to_string()),
-Url::from("https://apps.apple.com/us/app/microscope-journal/id507267328".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/ardan-labs-podcast/id1539547911".to_string()),
-Url::from("https://podcasts.apple.com/gb/podcast/thoughtworks-technology-podcast/id881136697?i=1000584890418".to_string()),
-Url::from("https://itunes.apple.com/us/app/adweek-magazine/id570229326?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/edsurge-on-air/id972239500#".to_string()),
-Url::from("https://apps.apple.com/de/app/bvg-fahrinfo-plus-berlin/id284971745".to_string()),
-Url::from("https://apps.apple.com/de/app/taxi-berlin/id443169338".to_string()),
-Url::from("https://apps.apple.com/de/app/going-local-berlin/id964438630".to_string()),
-Url::from("https://apps.apple.com/app/wearedevs-world-congress-22/id1625710834".to_string()),
-Url::from("https://apps.apple.com/us/app/about-berlin/id1425152714".to_string()),
-Url::from("https://geo.itunes.apple.com/us/app/serenity-relaxing-ambient/id1185371817?mt=12&at=1010ltPG".to_string()),
-Url::from("http://itunes.apple.com/app/id1238649962".to_string()),
-Url::from("https://support.apple.com/en-us/HT210380".to_string()),
-Url::from("https://www.apple.com/am/mac/m1".to_string()),
-Url::from("https://apps.apple.com/us/app/day-one-2-journal-notes/id1044867788".to_string()),
-Url::from("https://apps.apple.com/pt/app/go-i-pace/id1397794566".to_string()),
-Url::from("https://itunes.apple.com/gb/podcast/nature-podcast/id81934659?mt=2".to_string()),
-Url::from("https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6AATIntro.html".to_string()),
-Url::from("https://support.apple.com/kb/ph21411?locale=en_US".to_string()),
-Url::from("https://itunes.apple.com/us/app/lenovo-3d-catalog/id1164430025?mt=8".to_string()),
-Url::from("https://itunes.apple.com/app/id1290128888".to_string()),
-Url::from("https://itunes.apple.com/us/app/wrike/id890048871?mt=8".to_string()),
-Url::from("https://music.apple.com/gb/playlist/bbc-philharmonic-orchestra-of-the-year-2023-nominee/pl.5921732a341d42c99337a1a7cde3f5dd?at=1l3v8Ma&ct=playlist_bbc-philharmonic-orchestra-of-t&itscg=30200&itsct=toolbox_linkbuilder&ls=1&app=music".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=H5FWFAEZ86VNJR9Y25GG&pageLoadId=AqScAGXlLsH2Og2q&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=KGS4XCDCERWHBX904KTF&pageLoadId=BIjEUvwHhteyozzz&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=BG1JH1KFKCF86TW7K07Q&pageLoadId=48P5R1nie5laYcsR&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=XA9CGTJQQQ1532V31779&pageLoadId=FRA8JGqyFpDrcSTk&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=W8PGKFYPV3N1DPJRBSF4&pageLoadId=vhYn8vZlccfbXKXb&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=Y5T2MD9Q99EX901MRJ99&pageLoadId=teMLdIeMi7gtY2D0&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=6QGCKS6Q14B2570A9C6Y&pageLoadId=UadY8Hi2MO0XVEmH&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=20XMEQKV4KM399Z3DJZB&pageLoadId=BN9m4ApVmjfeH44O&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=3JMMYVAR6GKJXFXGDP27&pageLoadId=9SJJwwuJOBhbhzfI&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=RZMTNTNP5ES90TBZSYR8&pageLoadId=KCKIJ7L0Eyu0gPyk&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=TN13W97JDJ647QABA4EX&pageLoadId=q0MJhs7b2apczQAB&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=VKAMXR50EWHMB4JY77AC&pageLoadId=Acc4IyWt1DLl5QfM&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=5N50GSZX2S39GSVMR9PJ&pageLoadId=Zn07nkCtOGonOUfH&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=M5K4BK2KPTB7HA82T3N3&pageLoadId=sDYj4djOnGzMLb7j&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=EJB2JSMXK65J3RYWXPFB&pageLoadId=STrIRrqqustEa93k&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=PZYKXDK1QVNSY5FYGFPR&pageLoadId=tCq5Buyk860Wsvq7&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=J2CN9MEFXKCZJE5CNG1B&pageLoadId=Asy3WiZ6T3IJjEsu&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=ZCVE1TKH3CXEMWANVDXA&pageLoadId=Jaz5nrhH7zBNm33X&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=FEHE4CZJZFY4RKGZPGRS&pageLoadId=HY1aRBlNXutNeFGb&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=1V15C26X8B8C9EVPFRE5&pageLoadId=Lcf8tcXGjBQBFT8r&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=KZP2G549BDRS8DGVZVBN&pageLoadId=CqpUiRnw3DAcgm2V&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=NV4XYEXKB796YX2V0KWK&pageLoadId=6zsVgP3A475l073h&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=WN2BSG3GMWR2FGJTD790&pageLoadId=ELScE1ACkLd2oUS6&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=SSFF13ZDNNGKXC5NK3KE&pageLoadId=PgI0C7FfVLAfdQOi&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=df3314c2-582c-4e54-8eed-d5a4bf97c8ca&pf_rd_r=VYNYGWWXA5TD0JZE3812&pageLoadId=t0yablgDG2JNlBcm&creativeId=9adc166d-b28f-4dd5-bfce-d65bdc6a1a13".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=R1TF8MVA6Z9BHFK6HNSF&pageLoadId=JTgn1d4Qj3BTSPJj&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=KAC02X5CEXSN3T6FJ5W3&pageLoadId=pVNbYTHcSRmy9LsU&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=RMA8JG7DQF12C0ARS2E7&pageLoadId=PEZTCuR49ThJaYNh&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=DFCMWRZ7V68X5FN7JGRM&pageLoadId=1QgdRJO0F04rBxB4&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=165TX149S4EG0GDBJ0M9&pageLoadId=hWqe1UDMwDsii6W7&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=HEVQ06X0PSE73XJJVBFH&pageLoadId=uuXTam0qvCOtsPQF&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=366FZTR6NY1RPVNBENZE&pageLoadId=MkwORBurjRrBD971&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=NMMNFBVFCY6B34C0HNXW&pageLoadId=3PdqbgS3EdEUUJSU&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=1R524XYNPHAKM5B77PXD&pageLoadId=0vie0Jfx9N5FAFbL&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=HB0Z73RFR7W8QTYG5G9S&pageLoadId=4ZOnh5ioc3Waw1FS&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=15fe8fa9-606b-4ab6-be26-fffbd045d7bd&pf_rd_r=RPME3P7RF5N9N0FMGS3W&pageLoadId=erNIxgdVCZ96jQN0&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=8DSBE2NQEEEBS04BDYJN&pageLoadId=5RiIY5ACXO41alrU&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=F25TQKQWG9D0FH2Y7FXN&pageLoadId=ocGVVaBWzhoZjmg8&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=WTCRF4CKKXB1X7KPYCVA&pageLoadId=hOhQ65k2vK0d9SIN&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=HBGSB527KB2RZJ1WT5P1&pageLoadId=L2qQrbXVZkyUqr1d&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=AXYTRZA4PA7VNVSZGGGE&pageLoadId=yVy7b49EZPYG1Oij&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=X0EVR7SXRTDK67WNKX64&pageLoadId=PHHJzGzcBpcV3QiQ&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=DJH9066ABTZD7R1NDM2T&pageLoadId=BvMfERP3XO5ytOIi&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=W0NNQX1JYRFCARHHTVDH&pageLoadId=YLi76rAZB9d7uphe&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=34N4KGM51P71DRKQ19P7&pageLoadId=l5IbeFfrkFLvNVf4&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=85RG7R5NG853KQ8TV9HQ&pageLoadId=oR5sHmKaZlpYTxiR&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=V57XMRPA5VY6AXP2TFSG&pageLoadId=PmuPbx6SlMFSXPwv&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b".to_string()),
-Url::from("https://developer.apple.com/streaming".to_string()),
-Url::from("https://developer.apple.com/documentation/http_live_streaming/example_playlists_for_http_live_streaming/creating_a_multivariant_playlist".to_string()),
-Url::from("https://itunes.apple.com/es/app/paradores-hoteles-restaurantes/id960788695".to_string()),
-Url::from("https://apps.apple.com/us/app/rise-sleep-energy-tracker/id1453884781".to_string()),
-Url::from("https://apps.apple.com/us/app/rise-track-sleep-circadian/id1453884781".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/where-we-go-next/id1527944755".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/the-ai-podcast/id1186480811?mt=2&adbsc=social_20161220_68874946&adbid=811257941365882882&adbpl=tw&adbpr=61559439".to_string()),
-Url::from("https://developer.apple.com/augmented-reality".to_string()),
-Url::from("http://itunes.apple.com/us/app/wikihow-how-to-diy-survival/id309209200?mt=8".to_string()),
-Url::from("https://support.apple.com/en-us/HT206481".to_string()),
-Url::from("https://developer.apple.com/support/code-signing".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/view_controllers/adding_a_document_browser_to_your_app/presenting_selected_documents".to_string()),
-Url::from("https://developer.apple.com/ios/human-interface-guidelines/system-capabilities/quick-look".to_string()),
-Url::from("https://developer.apple.com/xcode/swiftui".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uidocumentbrowserviewcontrollerdelegate/2874187-documentbrowser".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uidocumentbrowserviewcontroller".to_string()),
-Url::from("https://developer.apple.com/documentation/fileprovider".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uidocument/1619978-contents".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/view_controllers/building_a_document_browser_based_app".to_string()),
-Url::from("https://developer.apple.com/library/content/documentation/Miscellaneous/Reference/UTIRef/Articles/System-DeclaredUniformTypeIdentifiers.html".to_string()),
-Url::from("https://itunes.apple.com/us/app/airstitch/id645714375?mt=8".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uidocument".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uidocument/1619988-save".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uidocument/1619979-init".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uidocument/1619992-filetype".to_string()),
-Url::from("https://developer.apple.com/library/content/documentation/General/Reference/InfoPlistKeyReference/Articles/iPhoneOSKeys.html#//apple_ref/doc/uid/TP40009252-SW37".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uidocument/1619982-documentstate".to_string()),
-Url::from("https://www.apple.com/macos/catalina-preview".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/view_controllers/adding_a_document_browser_to_your_app".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uidocument/1619971-load".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uiviewcontroller/1621380-present".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uidocument/1619977-open".to_string()),
-Url::from("https://developer.apple.com/library/content/documentation/General/Reference/InfoPlistKeyReference/Articles/iPhoneOSKeys.html#//apple_ref/doc/uid/TP40009252-SW20".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uidocumentbrowserviewcontrollerdelegate".to_string()),
-Url::from("https://apps.apple.com/us/app/c-by-ge/id1082828238?ls=1".to_string()),
-Url::from("https://music.apple.com/us/artist/dawn-margie-beaton/414795545".to_string()),
-Url::from("http://itunes.apple.com/de/app/openseamap/id495210783".to_string()),
-Url::from("http://itunes.apple.com/de/app/openseamap/id495210783?mt=8".to_string()),
-Url::from("https://apps.apple.com/us/app/3cx-webmeeting/id1039756959".to_string()),
-Url::from("https://apps.apple.com/us/app/3cxphone-15-for-3cx-phone/id992045982?ls=1".to_string()),
-Url::from("https://apps.apple.com/us/app/3cx/id992045982".to_string()),
-Url::from("https://apps.apple.com/us/app/mot%C3%AE/id1046122820?platform=iphone".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/involve-ai-bolstering-customer-success-with-ai-with/id1517181598?i=1000556587495".to_string()),
-Url::from("https://apps.apple.com/us/app/work-mobile/id1127865121".to_string()),
-Url::from("https://itunes.apple.com/us/app/gopayment/id324389392?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/smart-id/id1188304729?l=et&ls=1&mt=8".to_string()),
-Url::from("https://www.apple.com/ios".to_string()),
-Url::from("https://support.apple.com/kb/PH17191?locale=es_ES".to_string()),
-Url::from("https://itunes.apple.com/app/apple-store/id473512078?pt=356848&at=1l3vpXa&ct=hp&mt=8".to_string()),
-Url::from("https://support.apple.com/en-il/guide/safari/sfri11471/mac".to_string()),
-Url::from("https://support.apple.com/en-gb/guide/voiceover/vo14111/10/mac/12.0".to_string()),
-Url::from("https://support.apple.com/en-gb/guide/iphone/iph3e2e2281/ios".to_string()),
-Url::from("https://support.apple.com/en-gb/HT210417".to_string()),
-Url::from("https://support.apple.com/en-us/HT206175".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/clustered-conversations-by-weka/id1621362291".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/the-gamesindustry-biz-podcast/id1195829295".to_string()),
-Url::from("http://www.apple.com/quicktime/download".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/distributed-with-matt-mullenweg/id1463243282".to_string()),
-Url::from("https://itunes.apple.com/us/app/airvisual-real-time-forecast/id1048912974?ls=1&mt=8%27".to_string()),
-Url::from("https://itunes.apple.com/gb/app/degiro-beleggen/id1099174456?l=en&mt=8".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/episode-123-dr-martin-weihrauch-smart-in-media-digital/id1490210201?i=1000576089087".to_string()),
-Url::from("https://developer.apple.com/documentation/webkit/wkwebview".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/filmstemmer/id1682882422".to_string()),
-Url::from("https://apps.apple.com/app/id766443283?mt=8".to_string()),
-Url::from("https://apps.apple.com/us/app/tableau-mobile/id434633927".to_string()),
-Url::from("https://apps.apple.com/us/app/id1517276406".to_string()),
-Url::from("https://apps.apple.com/ch/app/sbb-mobile/id294855237".to_string()),
-Url::from("https://apps.apple.com/gb/app/eyeora-vr/id1501679625".to_string()),
-Url::from("https://podcasts.apple.com/de/podcast/t3n-catch-up/id1627024991".to_string()),
-Url::from("https://podcasts.apple.com/de/podcast/t3n-podcast-das-w%C3%B6chentliche-update-f%C3%BCr-digitale-pioniere/id1115601393".to_string()),
-Url::from("https://podcasts.apple.com/de/podcast/t3n-daily/id1643152920".to_string()),
-Url::from("https://developer.apple.com/forums/thread/650026".to_string()),
-Url::from("https://apps.apple.com/nl/app/wandelnet-nl-wandelroutes/id1463429180".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/it-happened-in-hollywood/id1437795866".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/richard-obrien-the-rocky-horror-picture-show/id1437795866?i=1000617840833".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/between-the-brackets-a-mediawiki-podcast/id1346062023?ls=1".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/the-agile-uprising-podcast/id1163230424?ls=1".to_string()),
-Url::from("http://itunes.apple.com/us/podcast/edward-tufte-art-and-science/id297775181".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/code-ish/id1457879893".to_string()),
-Url::from("https://apps.apple.com/us/app/google-adsense/id680739529".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/alphabet-scoop/id1359922601?mt=2".to_string()),
-Url::from("https://itunes.apple.com/app/apple-store/id626148774?pt=2050088&ct=Modal&mt=8".to_string()),
-Url::from("http://opensource.apple.com/source/dyld/dyld-210.2.3/src/dyld_stub_binder.s".to_string()),
-Url::from("https://itunes.apple.com/ca/app/bc-lions-exp/id1263298513?mt=8".to_string()),
-Url::from("https://itunes.apple.com/es/app/telegram-messenger/id686449807?mt=8".to_string()),
-Url::from("https://developer.apple.com/documentation/security/certificate_key_and_trust_services/trust?language=objc".to_string()),
-Url::from("https://support.apple.com/en-us/HT210176".to_string()),
-Url::from("https://developer.apple.com/documentation/security/secure_transport".to_string()),
-Url::from("https://developer.apple.com/library/archive/documentation/Darwin/Conceptual/KernelProgramming/BSD/BSD.html#//apple_ref/doc/uid/TP30000905-CH214-TPXREF101".to_string()),
-Url::from("https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/getsockopt.2.html".to_string()),
-Url::from("https://apps.apple.com/us/app/wallmine-stocks-bitcoin/id1458107499".to_string()),
-Url::from("https://apps.apple.com/us/app/dfds-ferries-cruises/id1447470946".to_string()),
-Url::from("https://itunes.apple.com/in/app/valley-air-district/id526195423".to_string()),
-Url::from("https://support.apple.com/en-gb/HT211094".to_string()),
-Url::from("https://itunes.apple.com/app/apple-store/id954157737?pt=49113808&ct=georgegarside.com%2Fapps&mt=8&at=1001lwpF".to_string()),
-Url::from("https://itunes.apple.com/app/apple-store/id989353027?pt=49113808&ct=georgegarside.com%2Fapps&mt=8&at=1001lwpF".to_string()),
-Url::from("http://www.apple.com/trailers/fox/thehappening".to_string()),
-Url::from("http://itunes.apple.com/gb/podcast/british-library-podcasts/id174455610?ign-mpt=uo=4".to_string()),
-Url::from("https://itunes.apple.com/us/app/apple-store/id535886823?pt".to_string()),
-Url::from("https://support.apple.com/it-it/guide/safari/sfri11471/mac".to_string()),
-Url::from("https://support.apple.com/en-ca/guide/safari/sfri11471/mac".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/foodist-with-darya-rose-ph.d/id1090954776?mt=2".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/foodist-with-darya-rose-ph-d/id1090954776".to_string()),
-Url::from("https://developer.apple.com/documentation/app_clips/creating_an_app_clip".to_string()),
-Url::from("https://developer.apple.com/documentation/bundleresources/information_property_list/nslocationwheninuseusagedescription".to_string()),
-Url::from("https://developer.apple.com/design/human-interface-guidelines/ios/system-capabilities/notifications".to_string()),
-Url::from("https://developer.apple.com/documentation/corelocation/choosing_the_authorization_level_for_location_services".to_string()),
-Url::from("https://developer.apple.com/documentation/usernotifications/unnotificationserviceextension".to_string()),
-Url::from("https://developer.apple.com/documentation/uikit/uiapplicationdelegate/1623013-application".to_string()),
-Url::from("https://developer.apple.com/documentation/app_clips/enabling_notifications_in_app_clips".to_string()),
-Url::from("https://developer.apple.com/library/archive/qa/qa1633/_index.html".to_string()),
-Url::from("https://developer.apple.com/documentation/bundleresources/information_property_list/nslocationalwaysusagedescription".to_string()),
-Url::from("https://support.apple.com/en-us/HT203609".to_string()),
-Url::from("https://developer.apple.com/documentation/usernotifications/asking_permission_to_use_notifications".to_string()),
-Url::from("https://itunes.apple.com/ch/app/india-air-quality/id988591878?l=en&mt=8".to_string()),
-Url::from("https://developer.apple.com/documentation/widgetkit/timelineprovider".to_string()),
-Url::from("https://developer.apple.com/videos/play/wwdc2020/10034".to_string()),
-Url::from("https://developer.apple.com/documentation/widgetkit/creating-a-widget-extension".to_string()),
-Url::from("https://support.apple.com/en-us/HT208309".to_string()),
-Url::from("https://support.apple.com/guide/shortcuts/add-import-questions-before-sharing-apdf330fd3a0/ios".to_string()),
-Url::from("https://support.apple.com/guide/shortcuts/welcome/ios".to_string()),
-Url::from("https://itunes.apple.com/us/app/google-photos-store-search/id962194608?mt=8".to_string()),
-Url::from("https://itunes.apple.com/app/id853371601".to_string()),
-Url::from("https://developer.apple.com/technologies/ios".to_string()),
-Url::from("https://itunes.apple.com/us/app/google-translate/id414706506?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/google-slides/id879478102?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/youtube/id544007664?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/google-docs/id842842640?mt=8".to_string()),
-Url::from("http://developer.apple.com/iphone/index.action".to_string()),
-Url::from("https://itunes.apple.com/us/app/google-sheets/id842849113?mt=8".to_string()),
-Url::from("https://itunes.apple.com/app/idid909319292".to_string()),
-Url::from("https://itunes.apple.com/app/id905060486".to_string()),
-Url::from("https://itunes.apple.com/us/app/google-play-music/id691797987?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/google-calendar/id909319292?mt=8".to_string()),
-Url::from("http://developer.apple.com/opensource".to_string()),
-Url::from("https://developer.apple.com/library/ios/documentation/CoreBluetooth/Reference/CBCentralManager_Class/index.html#//apple_ref/occ/instm/CBCentralManager/cancelPeripheralConnection:".to_string()),
-Url::from("https://developer.apple.com/library/prerelease/ios/documentation/CoreBluetooth/Reference/CBCentralManager_Class/index.html#//apple_ref/doc/constant_group/Central_Manager_Initialization_Options".to_string()),
-Url::from("https://developer.apple.com/library/prerelease/ios/documentation/CoreBluetooth/Reference/CBPeripheral_Class/index.html#//apple_ref/c/tdef/CBCharacteristicWriteType".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/brian-collins/id328074695?i=1000384718204&mt=2".to_string()),
-Url::from("https://apps.apple.com/us/app/syncfusion-succinctly-series/id1336430099".to_string()),
-Url::from("https://www.apple.com/uk/swift".to_string()),
-Url::from("https://apps.apple.com/nz/app/1-news/id329786380".to_string()),
-Url::from("https://apps.apple.com/app/apple-store/id1081530898".to_string()),
-Url::from("https://apps.apple.com/us/app/the-new-yorker-today/id1081530898".to_string()),
-Url::from("https://support.apple.com/en-us/HT202039".to_string()),
-Url::from("https://apps.apple.com/us/app/flipboard-latest-stories".to_string()),
-Url::from("https://apps.apple.com/us/app/the-new-yorker/id1081530898".to_string()),
-Url::from("https://apps.apple.com/be/app/go-i-pace/id1397794566?l=fr".to_string()),
-Url::from("https://apps.apple.com/be/app/go-i-pace/id1397794566?l=nl".to_string()),
-Url::from("https://apps.apple.com/app/apple-store/id1241040030?pt=64957&ct=blizzard-global-footer&mt=8".to_string()),
-Url::from("http://www.apple.com/macosx".to_string()),
-Url::from("https://itunes.apple.com/us/app/mht-to-webarchive/id838885505?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/css-tidy/id700229660?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/tcl-runner/id701474998?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/python-lint/id891162632?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/python-runner/id689176796?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/php-lint/id878546858?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/web-proxy/id721103033?mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/huo-che-ding-piao-zhu-shou/id494486644?ls=1&mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/socks-proxy/id1116965059?mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/set-file-icon/id787613939?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/copy-as-text/id869232291?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/html-tidy/id700229601?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/php-runner/id686855602?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/chinese-conversion/id720125982?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/no-trash-delete-files-without/id777213066?ls=1&mt=12".to_string()),
-Url::from("https://apps.apple.com/app/new-file-menu/id1064959555?l=en&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/webarchive-to-mht/id838885411?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/ip-messenger/id875939790?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/chmplus-chm-reader/id588628901?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/jslint/id757770685?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/simple-svn-server/id1040404148?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/epub-reader/id591299853?ls=1&mt=12".to_string()),
-Url::from("https://apps.apple.com/cn/story/id1520078797?l=en".to_string()),
-Url::from("https://itunes.apple.com/us/app/copy-as-html/id868049821?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/simple-hg-server/id1040403596?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/svn-status/id842480930?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/extractor-extract-rar-zip/id747644886?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/css-beautifier/id757811119?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/monitor-test/id904107000?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/favicon-creator/id787259510?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/simple-git-server/id1040403936?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/mht-viewer/id622609427?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/app/context-menu/id1236813619?l=en&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/copy-file-path/id777213048?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/hg-status/id993527882?mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/shell-checker/id892605935?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/paste-as-file/id825675368?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/archive-viewer/id598123939?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/dock-icons/id788702540?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/json-lint/id894042665?ls=1&mt=12".to_string()),
-Url::from("https://support.apple.com/kb/ht4094".to_string()),
-Url::from("https://itunes.apple.com/us/app/lua-runner/id689181261?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/icon-extractor/id803083078?ls=1&mt=12".to_string()),
-Url::from("https://apps.apple.com/app/context-menu/id1236813619?l=en&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/snailfossil/id1226366997?ls=1&mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/webarchive-to-html/id844587292?ls=1&mt=12".to_string()),
-Url::from("https://apps.apple.com/app/git-credential-manager/id1575689398?l=en&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/node-runner/id689175950?ls=1&mt=12".to_string()),
-Url::from("https://apps.apple.com/app/java-runner/id689184215".to_string()),
-Url::from("https://itunes.apple.com/us/app/perl-runner/id689178925?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/icon-converter/id803070906?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/html5-tidy/id700229688?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/startup-time/id875911449?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/app/chmplus-pro-chm-reader/id441521818?mt=8".to_string()),
-Url::from("https://apps.apple.com/app/filedrop-quick-file-transfer/id1605772776?l=en&mt=12".to_string()),
-Url::from("https://itunes.apple.com/app/webdav-server/id1185282799?l=en&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/git-status/id842481797?mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/new-file-here/id777221999?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/app/new-file-menu-free/id1066302071?l=en&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/chm-to-html/id869362084?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/share-clipboard/id619587463?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/ipagerank-pagerank-checker/id433671401?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/ruby-runner/id689179916?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/js-runner/id689183220?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/js-beautifier/id700229527?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/source-code-counter/id889599066?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/mht-to-html/id844591996?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/clipboard-viewer/id870366108?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/qr-clipboard/id869183896?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/chm-lite/id443454952?mt=8".to_string()),
-Url::from("https://itunes.apple.com/cn/artist/langui.net/id433671404?l=en&mt=12".to_string()),
-Url::from("https://itunes.apple.com/app/new-file-menu/id1064959555?l=en&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/snailhg/id1006394020?mt=12".to_string()),
-Url::from("https://itunes.apple.com/app/chmplus-lite-chm-reader/id443454952?mt=8".to_string()),
-Url::from("https://apps.apple.com/app/isvn/id861041982?ls=1".to_string()),
-Url::from("https://apps.apple.com/app/ftp-mounter/id1437045769?l=en&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/hg-server/id1034692383?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/git-server/id959332098?ls=1&mt=8".to_string()),
-Url::from("https://itunes.apple.com/cn/app/huo-che-ding-piao-zhu-shou/id494486644?mt=8".to_string()),
-Url::from("https://itunes.apple.com/app/chmplus-chm-reader/id441521818?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/network-speed-status/id875176096?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/chmplus-free-chm-reader/id554596300?ls=1&mt=8".to_string()),
-Url::from("https://itunes.apple.com/app/zipmounter/id1315374401?l=en&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/smartcopy/id884734329?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/svn-server/id685191052?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/sftp-server/id1300094971?ls=1&mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/css-lint/id757865011?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/us/app/new-file-menu/id1064959555?ls=1&mt=12".to_string()),
-Url::from("https://itunes.apple.com/nl/app/logius/id1167926346?ls=1&mt=8".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/e356-jeff-seibert-co-founder-at-digits/id1442272943?i=1000567828292".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/episode-2-fintech-what-the-heck-digits/id1578932483?i=1000568891111".to_string()),
-Url::from("https://podcasts.apple.com/us/podcast/digitalconomics-podcast/id1611260822".to_string()),
-Url::from("https://itunes.apple.com/us/app/ticketmaster/id500003565?mt=8".to_string()),
-Url::from("https://itunes.apple.com/us/app/bitfinex/id1436383182?ls=1&mt=8".to_string()),
-Url::from("http://support.apple.com/kb/PH17191?viewlocale=fr_FR".to_string()),
-Url::from("https://support.apple.com/ja-jp/HT202074".to_string()),
-Url::from("https://apps.apple.com/app/apple-store/id1625333066?pt=125060841&ct=website-main&mt=8".to_string()),
-Url::from("https://apps.apple.com/app/id1594445686".to_string()),
-Url::from("https://itunes.apple.com/app/id911410294".to_string()),
-Url::from("https://apps.apple.com/app/realplayer-mobile/id1578950397".to_string()),
-Url::from("https://apps.apple.com/us/app/trnio-plus-3d-scanner/id1546160586".to_string()),
-Url::from("https://apps.apple.com/us/app/trnio-plus-3d-scanner/id1546160586?itsct=apps_box_badge&itscg=30200".to_string()),
-Url::from("https://apps.apple.com/pt/app/3cx/id992045982".to_string()),
-Url::from("https://itunes.apple.com/gb/podcast/art-matters/id1343288276?mt=2".to_string()),
-Url::from("https://itunes.apple.com/cn/podcast/%E6%9E%81%E5%AE%A2%E7%94%B5%E5%8F%B0/id914427651?mt=2".to_string()),
-Url::from("https://support.apple.com/en-us/HT201859".to_string()),
-Url::from("https://itunes.apple.com/us/podcast/the-big-picture/id1439252196?mt=2".to_string()),
-Url::from("https://itunes.apple.com/us/app/uberfit-runner/id1033916715?mt=8&at=10lwCW&uo=8&at=10lwCW".to_string()),
-Url::from("https://www.apple.com/apple-music".to_string()),
-Url::from("https://www.apple.com/legal/internet-services/itunes/us/terms.html".to_string()),
+Url::parse("https://www.apple.com/newsroom/2023/06/developer-tools-to-create-spatial-experiences-for-apple-vision-pro-now-available").unwrap(),
+Url::parse("https://apps.apple.com/us/app/top10-com/id1545027369").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/safari-to-go/id881697395").unwrap(),
+Url::parse("https://support.apple.com/fr-fr/safari").unwrap(),
+Url::parse("https://apps.apple.com/app/id1145737091").unwrap(),
+Url::parse("https://itunes.apple.com/app/id473512078").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/articles/id364881979").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/geosearch-for-wikipedia/id929984054").unwrap(),
+Url::parse("https://itunes.apple.com/eg/app/wikiflip/id1156078290?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/app/wikiwand-faster-wikipedia-reader/id947444965").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/learn-discovery-mindmap-wikipedia/id859954749").unwrap(),
+Url::parse("https://apps.apple.com/us/app/duovocal-learn-german/id1548499329").unwrap(),
+Url::parse("https://itunes.apple.com/app/viki-wikipedia/id993435362").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/wikimed/id1281693200?mt=8").unwrap(),
+Url::parse("https://apps.apple.com/us/app/miniwiki/id1448143759#?platform=appleWatch").unwrap(),
+Url::parse("https://itunes.apple.com/app/wikicompass/id1363789167").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/inquire-by-tamper/id1045268097").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/kiwix/id997079563").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/wikinodes/id433834594").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/id1046122820").unwrap(),
+Url::parse("https://itunes.apple.com/en/app/wikipanion/id288349436").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/wikiweb-visual-wikipedia-reader/id499345601").unwrap(),
+Url::parse("https://itunes.apple.com/eg/app/wikimaster/id1062887157?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/tech-xplore/id1154953172?mt=8").unwrap(),
+Url::parse("https://developer.apple.com/documentation/xcode/writing-armv6-code-for-ios").unwrap(),
+Url::parse("https://apps.apple.com/us/app/polygit-git-client/id1508942822").unwrap(),
+Url::parse("https://apps.apple.com/ro/app/cinemagia-mobile/id445358827").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/id1082193172?mt=8").unwrap(),
+Url::parse("https://help.apple.com/systemimageutility/mac/10.12/#/sysmfd460053").unwrap(),
+Url::parse("https://support.apple.com/en-gb/HT208050").unwrap(),
+Url::parse("https://developer.apple.com/documentation/swiftdata/relationship").unwrap(),
+Url::parse("https://developer.apple.com/documentation/SwiftData").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/when-to-use-microservices-when-not-to-sam-newman-martin/id1559093928?i=1000514252180").unwrap(),
+Url::parse("http://support.apple.com/kb/HT5012").unwrap(),
+Url::parse("http://www.apple.com/certificateauthority/ca_program.html").unwrap(),
+Url::parse("http://support.apple.com/kb/HT6005").unwrap(),
+Url::parse("http://www.apple.com").unwrap(),
+Url::parse("http://support.apple.com/kb/HT1677?viewlocale=da_DK").unwrap(),
+Url::parse("http://maps.apple.com/?q=Highgate+Cemetery,+Swains+Lane,+London+N6+6PJ").unwrap(),
+Url::parse("http://itunes.apple.com/us/book/isbn9780990582915").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/git-drive/id1215974700?ls=1&mt=8").unwrap(),
+Url::parse("https://apple.com").unwrap(),
+Url::parse("https://www.apple.com/ca/osx/apps/app-store/index.html").unwrap(),
+Url::parse("https://support.apple.com/de-de/guide/safari/sfri11471/mac").unwrap(),
+Url::parse("https://support.apple.com/sv-se/guide/iphone/iph96b214f0/ios").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/wikipedia-mobile/id324715238?mt=8").unwrap(),
+Url::parse("https://testflight.apple.com/join/Z0AU0KXC").unwrap(),
+Url::parse("https://itunes.apple.com/us/institution/library-of-congress/id386017159").unwrap(),
+Url::parse("https://itunes.apple.com/us/artist/library-of-congress/id1280713133?mt=2").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/new-york-post-ipad-edition/id378590820?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/new-york-post-iphone-edition/id410094240?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/au/app/pocket-casts/id414834813").unwrap(),
+Url::parse("https://www.apple.com/apple-watch-ultra").unwrap(),
+Url::parse("http://support.apple.com/kb/PH5042").unwrap(),
+Url::parse("http://www.apple.com/macos").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT208331").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT205030").unwrap(),
+Url::parse("https://itunes.apple.com/us/artist/id2080713").unwrap(),
+Url::parse("https://www.apple.com/osx").unwrap(),
+Url::parse("http://support.apple.com/kb/HT4228").unwrap(),
+Url::parse("https://support.apple.com/en-ie/guide/safari/sfri11471/mac").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT201265").unwrap(),
+Url::parse("http://itunes.apple.com/us/app/whatsapp-messenger/id310633997?mt=8&fbclid=IwAR30kILqJposAMfPq2JiGQ00o5NwVDc3jvu86wtcmAwzPNCTJXLZaooUpVk").unwrap(),
+Url::parse("https://support.apple.com/downloads/#safari").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/the-identifiers/id564618183").unwrap(),
+Url::parse("https://apps.apple.com/app/packt/id1376577776?ls=1").unwrap(),
+Url::parse("https://appleid.apple.com/auth/authorize?redirect_uri=https%3A%2F%2Fwww.meetup.com%2Fties2%2F&state=returnUri%3Dhttps%253A%252F%252Fwww.meetup.com%252Fhome&client_id=com.meetup.web&response_type=code%20id_token&scope=name%20email&response_mode=form_post").unwrap(),
+Url::parse("https://apps.apple.com/us/app/meetup/id375990038").unwrap(),
+Url::parse("https://apple.com/mac").unwrap(),
+Url::parse("https://apps.apple.com/us/app/tamtam-messenger-video-calls/id1095345669").unwrap(),
+Url::parse("https://apps.apple.com/us/app/nicegram-ai-chat-for-telegram/id1608870673").unwrap(),
+Url::parse("https://apps.apple.com/app/telegram-messenger/id686449807?platform=iphone").unwrap(),
+Url::parse("https://apps.apple.com/us/developer/telegram-fz-llc/id686450210").unwrap(),
+Url::parse("https://support.apple.com/kb/PH2701").unwrap(),
+Url::parse("https://apps.apple.com/us/app/status-saver-video-photo-save/id1528272582").unwrap(),
+Url::parse("https://itunes.apple.com/us/genre/id6005").unwrap(),
+Url::parse("https://apps.apple.com/us/app/telegram-tools/id1621317550").unwrap(),
+Url::parse("https://apps.apple.com/us/app/telegram-messenger/id686449807").unwrap(),
+Url::parse("https://apps.apple.com/us/charts/iphone/social-networking-apps/6005").unwrap(),
+Url::parse("https://apps.apple.com/us/app/viber-messenger-chats-calls/id382617920").unwrap(),
+Url::parse("https://apps.apple.com/app/telegram-messenger/id686449807/www.apple.com/us/shop/goto/exitstore").unwrap(),
+Url::parse("https://apps.apple.com/app/telegram-messenger/id686449807/www.apple.com/us/shop/goto/home").unwrap(),
+Url::parse("https://apps.apple.com/us/app/dual-messenger-plus/id1530021020").unwrap(),
+Url::parse("https://apps.apple.com/app/telegram-messenger/id686449807?platform=ipad").unwrap(),
+Url::parse("https://apps.apple.com/us/app/fandom-videos-news-reviews/id1230063803").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/cold-call-by-harvard-business/id1156646189").unwrap(),
+Url::parse("https://vpp.itunes.apple.com").unwrap(),
+Url::parse("http://itunes.apple.com/us/app/chmplus-chm-reader/id441521818?mt=8").unwrap(),
+Url::parse("http://itunes.apple.com/cn/app/id441756221?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/app/snailgit-free-git-client-with-finder-integration/id1099475282?mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/snailgit/id847260112?l=en&mt=12").unwrap(),
+Url::parse("http://itunes.apple.com/app/ama-connect/id1355068050?uo=5&at=10l9yE").unwrap(),
+Url::parse("http://developer.apple.com/documentation/MacOSX/Conceptual/BPRuntimeConfig/index.html").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT202074").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/pytorch-developer-podcast/id1566080008").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/inaturalist/id421397028?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/app/apple-store/id736535961?pt=2334150&ct=Coursera%20Web%20Promo%20Banner&mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/privacy-management/id1365754887?mt=8").unwrap(),
+Url::parse("https://pcr.apple.com/id1264567358").unwrap(),
+Url::parse("https://support.apple.com/en-is/guide/iphone/iph4f4cbd242/14.0/ios/14.0").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/onlyoffice-documents/id944896972").unwrap(),
+Url::parse("http://support.apple.com/kb/HT1677").unwrap(),
+Url::parse("https://apps.apple.com/app/impact-com/id1629721141").unwrap(),
+Url::parse("https://apps.apple.com/app/id1671420139").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/alchemy/id1503608708").unwrap(),
+Url::parse("https://apps.apple.com/us/app/haaretz-english-edition/id504537897").unwrap(),
+Url::parse("https://apps.apple.com/us/app/microsoft-whiteboard/id1352499399").unwrap(),
+Url::parse("https://support.apple.com/en-gb/guide/apple-business-connect/abcbb749656b/web").unwrap(),
+Url::parse("https://www.apple.com/business/connect").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/id447919187?mt=8").unwrap(),
+Url::parse("https://apps.apple.com/us/app/id447919187?platform=ipad").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/wireguard/id1451685025?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/wireguard/id1441195209?ls=1&mt=8").unwrap(),
+Url::parse("https://apps.apple.com/us/app/geekbench-ml/id1563487010").unwrap(),
+Url::parse("https://apps.apple.com/us/app/geekbench-6/id1565728895").unwrap(),
+Url::parse("https://www.apple.com/wss/fonts?families=SF+Pro,v1|SF+Pro+Icons,v1").unwrap(),
+Url::parse("https://www.apple.com/privacy").unwrap(),
+Url::parse("https://www.apple.com/news").unwrap(),
+Url::parse("https://apps.apple.com/us/app/id1460576753").unwrap(),
+Url::parse("https://apps.apple.com/fr/app/engie-gaz-et-electricit%C3%A9/id1104269908").unwrap(),
+Url::parse("https://apps.apple.com/us/app/bloomberg-business-news/id281941097").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/glassdoor-job-search-salaries/id589698942?mt=8&l=eng").unwrap(),
+Url::parse("https://itunes.apple.com/app/id980888073").unwrap(),
+Url::parse("https://support.apple.com/en-gb/guide/safari/sfri11471/mac").unwrap(),
+Url::parse("https://support.apple.com/en-gb/HT201265").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/99-invisible/id394775318?mt=2").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/99-invisible/id394775318?at=10l9zE").unwrap(),
+Url::parse("https://tv.apple.com/vn/show/acapulco/umc.cmc.8xrkg9zywke7g6a9ahmvpr3l").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2022/04/apple-unveils-the-best-photos-from-the-shot-on-iphone-macro-challenge").unwrap(),
+Url::parse("https://support.apple.com/guide/safari/customize-website-notifications-sfri40734/16.1/mac/13.0").unwrap(),
+Url::parse("http://www.apple.com/safari").unwrap(),
+Url::parse("https://apps.apple.com/app/id1622656471").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/ip2location-geolocation/id771597553?ls=1&mt=8").unwrap(),
+Url::parse("https://apps.apple.com/es/app/diario-abc/id336860348").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=PAMKMDHYQRZYRVK1T2SN&pageLoadId=PBbn8FzEc7Zkrq58&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://apps.apple.com/us/app/nyt-audio/id1549293936").unwrap(),
+Url::parse("https://itunes.apple.com/si/app/mcobiss/id687682312?mt=8").unwrap(),
+Url::parse("https://support.apple.com/en-gb/accessibility").unwrap(),
+Url::parse("https://maps.apple.com/?q=48.8,2.3&t=k").unwrap(),
+Url::parse("https://maps.apple.com/?q=48.8,2.3&t=m").unwrap(),
+Url::parse("https://www.apple.com/legal/privacy/en-ww/cookies").unwrap(),
+Url::parse("https://www.apple.com/legal/privacy/en-ww").unwrap(),
+Url::parse("https://itunes.apple.com/ru/app/id1064581926").unwrap(),
+Url::parse("https://apps.apple.com/app/apple-store/id425194759?pt=417731&ct=enterprise_mobile_comms_LP&mt=8").unwrap(),
+Url::parse("https://apps.apple.com/app/apple-store/id1356686763?pt=417731&ct=lohp&mt=8").unwrap(),
+Url::parse("https://apps.apple.com/us/app/vimeo-create/id1491791513").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/firefox-private-safe-browser/id989804926").unwrap(),
+Url::parse("https://support.apple.com/guide/security/controlling-app-access-to-files-secddd1d86a6/web").unwrap(),
+Url::parse("https://apps.apple.com/us/app/stc-training-club/id575468749").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/what-we-learned-today/id1465904347").unwrap(),
+Url::parse("https://itunes.apple.com/app/apple-store/id1036449551?pt=117888292&ct=discogs-app-page&mt=8").unwrap(),
+Url::parse("https://developer.apple.com/downloads").unwrap(),
+Url::parse("https://developer.apple.com/technologies/tools").unwrap(),
+Url::parse("https://developer.apple.com/macosx/launchd.html").unwrap(),
+Url::parse("https://feedbackassistant.apple.com").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/xcode/id497799835").unwrap(),
+Url::parse("https://developer.apple.com/downloads/index.action").unwrap(),
+Url::parse("https://www.apple.com/itunes").unwrap(),
+Url::parse("https://support.apple.com/downloads#quicktime").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/software-engineering-institute-sei-podcast-series/id566573552?mt=2").unwrap(),
+Url::parse("https://itunes.apple.com/podcast/stephan-livera-podcast/id1415720320?mt=2").unwrap(),
+Url::parse("https://music.apple.com/za/post/1513061930").unwrap(),
+Url::parse("https://support.apple.com/kb/PH17191?locale=it_IT").unwrap(),
+Url::parse("https://apps.apple.com/us/app/the-wall-street-journal/id364387007?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/my-tune-device/id1100377074?ls=1&mt=8").unwrap(),
+Url::parse("https://apps.apple.com/ie/developer/the-irish-times-ltd/id365544597").unwrap(),
+Url::parse("https://itunes.apple.com/us/book/id1457507869").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/cctvnews/id922456579").unwrap(),
+Url::parse("https://support.apple.com/kb/PH21411?locale=en_US").unwrap(),
+Url::parse("https://support.apple.com/kb/PH5042?locale=en_US").unwrap(),
+Url::parse("https://support.apple.com/guide/apple-configurator-2/revive-or-restore-a-mac-with-apple-silicon-apdd5f3c75ad/mac").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/propublica/id355298887?mt=8").unwrap(),
+Url::parse("https://apps.apple.com/app/duckduckgo-privacy-browser/id663592361?ref=spreadprivacy.com").unwrap(),
+Url::parse("https://apps.apple.com/us/app/duckduckgo-privacy-browser/id663592361?ref=spreadprivacy.com").unwrap(),
+Url::parse("https://apps.apple.com/us/app/hubspot/id1107711722").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT204915").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT202303").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/effs-how-to-fix-the-internet/id1539719568").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT201060").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2022/12/apple-advances-user-security-with-powerful-new-data-protections").unwrap(),
+Url::parse("https://developer.apple.com/find-my").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT201088").unwrap(),
+Url::parse("https://apps.apple.com/us/app/remote-desktop-mobile/id714464092").unwrap(),
+Url::parse("https://itunes.apple.com/de/podcast/filterblase-t3n-pioneers-podcast/id1115601393?mt=2").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/an-arm-and-a-leg/id1438778444").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/american-diagnosis-with-dr-celine-gounder/id1282044849?uo=4").unwrap(),
+Url::parse("https://podcasts.apple.com/podcast/where-it-hurts/id1529977695?mt=2").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/what-the-health/id1253607372?mt=2").unwrap(),
+Url::parse("https://apps.apple.com/jp/app/amazon-seller/id794141485").unwrap(),
+Url::parse("https://developer.apple.com/design/tips").unwrap(),
+Url::parse("https://developer.apple.com/library/prerelease/ios//documentation/AVFoundation/Reference/AVCaptureSession_Class/index.html").unwrap(),
+Url::parse("https://developer.apple.com/library/archive/documentation/FileManagement/Conceptual/DocumentPickerProgrammingGuide/AccessingDocuments/AccessingDocuments.html").unwrap(),
+Url::parse("https://developer.apple.com/videos/wwdc/2012").unwrap(),
+Url::parse("https://developer.apple.com/videos/play/wwdc2016/413").unwrap(),
+Url::parse("https://developer.apple.com/documentation/appkit/nsstandardkeybindingresponding?language=objc").unwrap(),
+Url::parse("https://developer.apple.com/documentation/foundation/nsdata/3174960-compressed").unwrap(),
+Url::parse("https://support.apple.com/en-gb/guide/preview/prvw587dd90f/mac").unwrap(),
+Url::parse("https://developer.apple.com/reference/xctest").unwrap(),
+Url::parse("https://developer.apple.com/documentation/mapkit/mkgeojsondecoder").unwrap(),
+Url::parse("https://support.apple.com/en-gb/guide/mac-help/mchlp1560/mac").unwrap(),
+Url::parse("https://developer.apple.com/sf-symbols").unwrap(),
+Url::parse("https://support.apple.com/en-gb/guide/mac-help/mchlp1046/mac").unwrap(),
+Url::parse("https://developer.apple.com/documentation/appkit/nsview/1483440-bitmapimagerepforcachingdisplay").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/IDEs/Conceptual/AppDistributionGuide/MaintainingProfiles/MaintainingProfiles.html").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/qa/qa1633/_index.html").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/LanguagesUtilities/Conceptual/iTunesConnect_Guide/Chapters/CreatingiTunesConnectRecord.html").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/userexperience/Conceptual/MobileHIG/index.html#//apple_ref/doc/uid/TP40006556").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/IDEs/Conceptual/AppDistributionGuide/TestingYouriOSApp/TestingYouriOSApp.html#//apple_ref/doc/uid/TP40012582-CH8-SW1").unwrap(),
+Url::parse("https://developer.apple.com/search").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/StringsTextFonts/Conceptual/TextAndWebiPhoneOS/UsingTextClasses/UsingTextClasses.html").unwrap(),
+Url::parse("https://developer.apple.com/library/archive/technotes/tn2339/_index.html").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/UserExperience/Conceptual/MobileHIG/IconMatrix.html").unwrap(),
+Url::parse("https://developer.apple.com/documentation/imageio/3333271-cganimateimageaturlwithblock").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619945-statechangednotification").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/UIKit/Reference/UIActivityViewController_Class").unwrap(),
+Url::parse("https://developer.apple.com/documentation/swift_packages/supportedplatform/3788290-maccatalyst").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/view_controllers/preserving_your_app_s_ui_across_launches").unwrap(),
+Url::parse("https://books.apple.com/us/book/id1551005489").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/StringsTextFonts/Conceptual/TextAndWebiPhoneOS/CustomTextProcessing/CustomTextProcessing.html").unwrap(),
+Url::parse("https://developer.apple.com/download/more").unwrap(),
+Url::parse("https://apps.apple.com/gb/app/scratch/id1671420139").unwrap(),
+Url::parse("https://developer.apple.com/documentation/security/password_autofill/customizing_password_autofill_rules").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/general/conceptual/CocoaEncyclopedia/Introduction/Introduction.html#//apple_ref/doc/uid/TP40010810-CH1-SW1").unwrap(),
+Url::parse("https://itunes.apple.com/gb/course/developing-ios-7-apps-for/id733644550").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/samplecode/PhotoScroller/Introduction/Intro.html").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/general/conceptual/CocoaEncyclopedia/Initialization/Initialization.html#//apple_ref/doc/uid/TP40010810-CH6-SW3").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uicollectionviewcompositionallayout").unwrap(),
+Url::parse("https://developer.apple.com/videos/play/wwdc2015/219").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/General/Conceptual/DevPedia-CocoaCore/AppID.html").unwrap(),
+Url::parse("https://support.apple.com/en-bw/guide/activity-monitor/actmaea30277/mac").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/UserExperience/Conceptual/AutolayoutPG/VisualFormatLanguage.html").unwrap(),
+Url::parse("https://apps.apple.com/app/id1558358855").unwrap(),
+Url::parse("https://apps.apple.com/us/app/rested-simple-http-requests/id421879749").unwrap(),
+Url::parse("https://developer.apple.com/documentation/swiftui/environmentvalues").unwrap(),
+Url::parse("https://developer.apple.com/swift/blog/?id=25").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html#//apple_ref/doc/uid/TP40009249-SW10").unwrap(),
+Url::parse("https://developer.apple.com/documentation/swiftui/state_and_data_flow").unwrap(),
+Url::parse("https://support.apple.com/en-gb/HT206481#thirdparty").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/technotes/tn2259/_index.html").unwrap(),
+Url::parse("https://developer.apple.com/videos/wwdc/2014").unwrap(),
+Url::parse("https://testflight.apple.com").unwrap(),
+Url::parse("https://developer.apple.com/documentation/Charts").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/UserExperience/Conceptual/AutolayoutPG/AdoptingAutoLayout/AdoptingAutoLayout.html").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/Cocoa/Conceptual/ProgrammingWithObjectiveC/WorkingwithBlocks/WorkingwithBlocks.html").unwrap(),
+Url::parse("https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/CodingGuidelines/Articles/NamingMethods.html#//apple_ref/doc/uid/20001282-BCIGIJJF").unwrap(),
+Url::parse("https://apps.apple.com/app/id1578965002").unwrap(),
+Url::parse("https://developer.apple.com/documentation/charts/linemark").unwrap(),
+Url::parse("https://apps.apple.com/app/id1558358855?action=write-review").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/qa/qa1713/_index.html").unwrap(),
+Url::parse("https://www.apple.com/legal/internet-services/itunes/testflight/sren/terms.html").unwrap(),
+Url::parse("https://developer.apple.com/library/mac/releasenotes/ObjectiveC/RN-TransitioningToARC/Introduction/Introduction.html").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/AVFoundation/Reference/AVCaptureVideoPreviewLayer_Class/index.html").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/uncast/id1566634831").unwrap(),
+Url::parse("https://apps.apple.com/us/app/fliff-sports-picks/id1489145500?see-all=reviews").unwrap(),
+Url::parse("https://apps.apple.com/us/app/fliff/id1489145500?ls=1").unwrap(),
+Url::parse("https://support.apple.com/en-gb/HT205280").unwrap(),
+Url::parse("https://developer.apple.com/reference/appkit/nsaccessibility").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/between-the-brackets-a-mediawiki-podcast/id1346062023?mt=2").unwrap(),
+Url::parse("https://support.apple.com/guide/safari/manage-cookies-sfri11471/mac").unwrap(),
+Url::parse("https://developer.apple.com/videos/play/wwdc2023/10121").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/the-stack-overflow-podcast/id1033688462").unwrap(),
+Url::parse("https://apps.apple.com/au/app/amazon-seller/id794141485").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2023/05/apple-launches-20-fun-new-games-for-its-award-winning-apple-arcade-service").unwrap(),
+Url::parse("https://apps.apple.com/us/app/brave-private-web-browser/id1052879175?mt=8&ign-mpt=uo%3D4").unwrap(),
+Url::parse("https://support.apple.com/en-il/guide/safari/manage-cookies-and-website-data-sfri11471/mac").unwrap(),
+Url::parse("https://apps.apple.com/app/google-chrome").unwrap(),
+Url::parse("https://developer.apple.com/documentation/hypervisor").unwrap(),
+Url::parse("https://support.apple.com/guide/safari/manage-cookies-and-website-data-sfri11471/14.0/mac/11.0").unwrap(),
+Url::parse("https://developer.apple.com/reference/foundation/nsurl/1417051-startaccessingsecurityscopedreso?language=objc").unwrap(),
+Url::parse("https://developer.apple.com/documentation/coreservices/1441725-lscopydefaulthandlerforurlscheme?language=objc").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/Foundation/Reference/NSUserActivity_Class/index.html#//apple_ref/occ/instp/NSUserActivity/activityType").unwrap(),
+Url::parse("https://developer.apple.com/documentation/foundation/nsusernotification").unwrap(),
+Url::parse("https://developer.apple.com/design/human-interface-guidelines/notifications").unwrap(),
+Url::parse("https://developer.apple.com/documentation/appkit/nsaboutpaneloptioncredits?language=objc").unwrap(),
+Url::parse("https://developer.apple.com/library/content/documentation/General/Reference/InfoPlistKeyReference/Articles/CocoaKeys.html#//apple_ref/doc/uid/TP40009251-SW35").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/UserExperience/Conceptual/Handoff/HandoffFundamentals/HandoffFundamentals.html").unwrap(),
+Url::parse("https://itunes.apple.com/gb/app/fudget-budget-planner-personal-finance-tracker/id1211707701?mt=12").unwrap(),
+Url::parse("https://developer.apple.com/library/content/documentation/Security/Conceptual/AppSandboxDesignGuide/AppSandboxInDepth/AppSandboxInDepth.html#//apple_ref/doc/uid/TP40011183-CH3-SW16").unwrap(),
+Url::parse("https://developer.apple.com/library/mac/documentation/Cocoa/Conceptual/EventOverview/EventArchitecture/EventArchitecture.html#//apple_ref/doc/uid/10000060i-CH3-SW7").unwrap(),
+Url::parse("https://developer.apple.com/documentation/appkit/nsapplication/1428608-servicesmenu?language=objc").unwrap(),
+Url::parse("https://developer.apple.com/documentation/usernotifications/unnotificationresponse").unwrap(),
+Url::parse("https://developer.apple.com/library/archive/technotes/tn2150/_index.html").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/General/Reference/InfoPlistKeyReference/Articles/CoreFoundationKeys.html#//apple_ref/doc/uid/TP40009249-102207-TPXREF115").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/general/Reference/InfoPlistKeyReference/Articles/AboutInformationPropertyListFiles.html").unwrap(),
+Url::parse("https://developer.apple.com/reference/appkit/nsapplication/1428479-orderfrontstandardaboutpanelwith?language=objc").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/you-are-not-so-smart-attention/id521594713?i=113727803").unwrap(),
+Url::parse("https://apps.apple.com/us/app/bloomberg-b-unit/id1344778873").unwrap(),
+Url::parse("https://apps.apple.com/app/flat/id1598891661").unwrap(),
+Url::parse("https://support.apple.com/es-es/HT201265").unwrap(),
+Url::parse("https://support.apple.com/kb/PH19214?viewlocale=es_ES&locale=es_ES").unwrap(),
+Url::parse("https://apps.apple.com/us/app/geo-measure-area-calculator/id1099413961").unwrap(),
+Url::parse("https://apps.apple.com/us/app/kml-viewer-and-converter/id1457028727").unwrap(),
+Url::parse("https://developer.apple.com/documentation/safari-release-notes/safari-16_1-release-notes").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/css-tricks-screencasts/id273881728").unwrap(),
+Url::parse("https://developer.apple.com/passkeys").unwrap(),
+Url::parse("https://www.apple.com/accessibility/vision").unwrap(),
+Url::parse("https://podcasts.apple.com/th/podcast/143-fighting-the-climate-crisis-with-data/id1336150688?i=1000618315641").unwrap(),
+Url::parse("https://apps.apple.com/us/app/permanent-archive/id1571883070").unwrap(),
+Url::parse("https://support.apple.com/en-gb/guide/safari/sfri35610/mac").unwrap(),
+Url::parse("https://support.apple.com/HT204416").unwrap(),
+Url::parse("https://itunes.apple.com/fr/app/glassdoor-job-search-salaries/id589698942?mt=8&l=fra").unwrap(),
+Url::parse("https://itunes.apple.com/nl/app/glassdoor-job-search-salaries/id589698942?mt=8&l=nld").unwrap(),
+Url::parse("https://apps.apple.com/cz/app/id766443283?l=cs&mt=8").unwrap(),
+Url::parse("https://support.apple.com/en-euro/guide/safari/sfri11471/mac").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/cdl-test-genie/id1447415467?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/dmv-genius-permit-practice/id513850893?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/app/apple-store/id1145737091").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/famous-birthdays-celebrity/id646707938").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/disconnect-privacy-pro-entire/id1057771839?ls=1&mt=8").unwrap(),
+Url::parse("https://www.apple.com/macbook-air-13-and-15-m2").unwrap(),
+Url::parse("https://www.apple.com/mac-pro").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT210748").unwrap(),
+Url::parse("https://www.apple.com/mac-studio").unwrap(),
+Url::parse("https://support.apple.com/guide/mac-help/calibrate-your-display-mchlp1109/mac").unwrap(),
+Url::parse("https://developer.apple.com/documentation/swiftdata").unwrap(),
+Url::parse("https://apps.apple.com/us/app/uptime-robot-app/id1104878581?utm_source=homepage-section&utm_campaign=website-app-promo&ct=homepage-section-below").unwrap(),
+Url::parse("https://apps.apple.com/us/app/uptime-robot-app/id1104878581?utm_source=homepage-section&utm_campaign=website-app-promo&ct=homepage-section").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/lmax-digital-trading/id1375900638").unwrap(),
+Url::parse("http://www.apple.com/fr/safari").unwrap(),
+Url::parse("https://support.apple.com/guide/safari/manage-cookies-and-website-data-sfri11471").unwrap(),
+Url::parse("https://support.apple.com/da-dk/HT201265").unwrap(),
+Url::parse("https://support.apple.com/da-dk/guide/safari/sfri11471/mac").unwrap(),
+Url::parse("https://developer.apple.com/documentation/quartzcore").unwrap(),
+Url::parse("https://apps.apple.com/mx/app/scotiam%C3%B3vil-mx/id720812080").unwrap(),
+Url::parse("https://developer.apple.com/documentation/virtualization/running_intel_binaries_in_linux_vms_with_rosetta").unwrap(),
+Url::parse("https://apps.apple.com/app/sketch-view-and-mirror/id1609224699").unwrap(),
+Url::parse("http://itunes.apple.com/us/app/faonow/id877618174?ls=1&mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/phys-org/id356443503?mt=8").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/programming-love-with-martin-kleppmann/id1518407590?i=1000495317576").unwrap(),
+Url::parse("https://podcasts.apple.com/ph/podcast/designing-data-intensive-applications-with-martin/id1531450276?i=1000533284011").unwrap(),
+Url::parse("https://apps.apple.com/app/datadog/id1391380318").unwrap(),
+Url::parse("https://help.apple.com/itc/podcasts_connect/#/itcbaf351599").unwrap(),
+Url::parse("https://help.apple.com/itc/podcasts_connect/#/itc2b3780e76").unwrap(),
+Url::parse("https://help.apple.com/itc/podcasts_connect/#/itcb54353390").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT201762").unwrap(),
+Url::parse("https://podcastsconnect.apple.com").unwrap(),
+Url::parse("https://appleid.apple.com/account#!&page=create").unwrap(),
+Url::parse("https://apps.apple.com/app/time.is/id857189816").unwrap(),
+Url::parse("https://podcasts.apple.com/bg/podcast/the-recursive-podcast/id1576538398").unwrap(),
+Url::parse("https://apps.apple.com/jp/app/sail-japanese-conversations/id1434041945").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2020/05/apple-to-host-virtual-worldwide-developers-conference-beginning-june-22").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/true-stories-of-good-people/id1389955443").unwrap(),
+Url::parse("https://support.apple.com/kb/PH21411?locale=nl_NL&viewlocale=en_US").unwrap(),
+Url::parse("https://support.apple.com/kb/PH21411?locale=nl_NL&viewlocale=fr_FR").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/shadertoy/id717961814").unwrap(),
+Url::parse("https://itunes.apple.com/it/app/la-repubblica-mobile/id313084825?mt=8").unwrap(),
+Url::parse("https://apps.apple.com/us/developer/wolfram-group-llc/id715130979").unwrap(),
+Url::parse("https://apps.apple.com/us/app/wolfram-alpha/id548861535").unwrap(),
+Url::parse("https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/PrioritizeWorkAtTheTaskLevel.html").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/jewish-virtual-library/id1336516435?mt=8").unwrap(),
+Url::parse("https://apps.apple.com/ca/app/snikket/id1545164189").unwrap(),
+Url::parse("https://developer.apple.com/wwdc").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/what-the-health/id1253607372?mt=2").unwrap(),
+Url::parse("https://apps.apple.com/us/app/uber/id368677368?uclick_id=c33a8d2c-1390-4ad3-9c63-bb9b1997e716").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/privacy-monitor/id1450531857?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/privacy-monitor/id1451670029?mt=12").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT205223").unwrap(),
+Url::parse("https://opensource.apple.com/source/xnu/xnu-7195.81.3/EXTERNAL_HEADERS/mach-o/fixup-chains.h.auto.html").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/kvikmyndir.is/id1191288272?ls=1&mt=8").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT202663").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/cold-call/id1156646189").unwrap(),
+Url::parse("https://www.apple.com/de/macos").unwrap(),
+Url::parse("https://developer.apple.com/opensource").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2023/06/ios-17-makes-iphone-more-personal-and-intuitive").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2023/06/tvos-17-brings-facetime-and-video-conferencing-to-apple-tv-4k").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2023/06/apple-introduces-the-15-inch-macbook-air").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2023/06/introducing-watchos-10-a-milestone-update-for-apple-watch").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2023/06/ipados-17-brings-new-levels-of-personalization-and-versatility-to-ipad").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2023/06/apple-introduces-m2-ultra").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2023/06/macos-sonoma-brings-new-capabilities-for-elevating-productivity-and-creativity").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT212850").unwrap(),
+Url::parse("https://developer.apple.com/homekit").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2023/06/apple-announces-powerful-new-privacy-and-security-features").unwrap(),
+Url::parse("https://developer.apple.com/videos/play/wwdc2023/10120").unwrap(),
+Url::parse("https://apps.apple.com/app/lmax-global-trading/id884042608").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/damn-interesting/id571835813").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/damn-interesting/id571835813?mt=2").unwrap(),
+Url::parse("https://itunes.apple.com/ro/app/iaBilet/id899564929").unwrap(),
+Url::parse("https://apps.apple.com/app/id6444779277").unwrap(),
+Url::parse("https://podcasts.apple.com/dk/podcast/datatilsynet-podcast-bliv-klogere-p%C3%A5-gdpr/id1481257016").unwrap(),
+Url::parse("https://itunes.apple.com/pl/app/podr%C4%99czny-s%C5%82ownik-hiszpa%C5%84sko-polski-pwn/id1120160080?mt=8&utm_source=ref&utm_medium=web&utm_campaign=6P_0916_ref_201609").unwrap(),
+Url::parse("https://tv.apple.com/movie/evil-dead-rise/umc.cmc.1qv727yx0ev1kdt33soi1xbmy?uo=5&itsct=tv_box_link&itscg=30200&at=1010l369s").unwrap(),
+Url::parse("https://tv.apple.com/movie/spider-man-into-the-spider-verse/umc.cmc.4a2ypxddplsdsahwxrc5fkz4x?uo=5&itsct=tv_box_link&itscg=30200&at=1010l369s").unwrap(),
+Url::parse("https://apps.apple.com/app/id980888073#?platform=iphone").unwrap(),
+Url::parse("https://apps.apple.com/app/id980888073#?platform=mac").unwrap(),
+Url::parse("https://apps.apple.com/gb/app/microsoft-office/id541164041").unwrap(),
+Url::parse("https://apps.apple.com/gb/app/microsoft-teams/id1113153706").unwrap(),
+Url::parse("https://apps.apple.com/us/app/skype-for-iphone/id304878510").unwrap(),
+Url::parse("https://apps.apple.com/us/app/shopify-inbox/id1301681854").unwrap(),
+Url::parse("https://apps.apple.com/us/app/facebook-ads-manager/id964397083").unwrap(),
+Url::parse("https://apps.apple.com/us/app/wix-owner-website-builder/id1545924344").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/xcode/id497799835?mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/wix-com/id1099748482").unwrap(),
+Url::parse("https://apps.apple.com/us/developer/wix-com-inc/id407141669").unwrap(),
+Url::parse("https://apps.apple.com/gb/app/xbox-game-pass/id1374542474").unwrap(),
+Url::parse("https://apps.apple.com/us/app/oculus/id1366478176").unwrap(),
+Url::parse("https://apps.apple.com/ca/app/shop-package-order-tracker/id1223471316").unwrap(),
+Url::parse("https://apps.apple.com/us/app/microsoft-outlook/id951937596").unwrap(),
+Url::parse("https://apps.apple.com/app/facebook/id284882215").unwrap(),
+Url::parse("https://apps.apple.com/us/app/shopify-point-of-sale-pos/id686830644").unwrap(),
+Url::parse("https://apps.apple.com/us/app/dine-by-wix/id1561524113").unwrap(),
+Url::parse("https://apps.apple.com/us/app/shopify-your-ecommerce-store/id371294472").unwrap(),
+Url::parse("https://support.apple.com/fr-afri/guide/safari/sfri11471/mac").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/swann-auction-galleries/id1440439183?ls=1&mt=8").unwrap(),
+Url::parse("http://support.apple.com/kb/DL1251").unwrap(),
+Url::parse("https://support.apple.com/kb/DL1283").unwrap(),
+Url::parse("http://opensource.apple.com/source/IOUSBFamily/IOUSBFamily-458.4.1/AppleUSBOHCI/Classes/AppleUSBOHCI_UIM.cpp").unwrap(),
+Url::parse("https://discussions.apple.com/thread/123054?start=0&tstart=0").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/the-photo-detective/id1255965884?mt=2&mc_cid=67037096ee&mc_eid=[UNIQID]").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/the-photo-detective/id1255965884?mt=2").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/audubon-bird-guide-north-america/id333227386?mt=8&utm_source=bird-guide&utm_medium=national&utm_campaign=iosapp").unwrap(),
+Url::parse("https://apps.apple.com/us/app/clips-clipboard-manager/id1549350243").unwrap(),
+Url::parse("https://store.apple.com/xc/product/MJYT2AM/A").unwrap(),
+Url::parse("https://apps.apple.com/us/app/obscura-3-pro-camera/id1579306989").unwrap(),
+Url::parse("https://apps.apple.com/us/app/halide-mark-ii-pro-camera/id885697368").unwrap(),
+Url::parse("https://developer.apple.com/documentation/xcode-release-notes").unwrap(),
+Url::parse("http://developer.apple.com").unwrap(),
+Url::parse("http://apple.com").unwrap(),
+Url::parse("https://developer.apple.com/library/mac/#documentation/Cocoa/Conceptual/MemoryMgmt/Articles/mmRules.html").unwrap(),
+Url::parse("https://apps.apple.com/us/app/keenetic/id1413650865").unwrap(),
+Url::parse("http://support.apple.com/kb/PH5067").unwrap(),
+Url::parse("https://www.apple.com/privacy/privacy-policy").unwrap(),
+Url::parse("http://developer.apple.com/documentation/Carbon/Conceptual/CoreText_Programming/Overview/chapter_2_section_4.html#//apple_ref/doc/uid/TP40005533-CH3-DontLinkElementID_10").unwrap(),
+Url::parse("http://www.apple.com/macosx/features/bootcamp.html").unwrap(),
+Url::parse("https://apps.apple.com/it/app/vanity-fair-italia/id539933331").unwrap(),
+Url::parse("https://support.apple.com/kb/PH19214").unwrap(),
+Url::parse("https://support.apple.com/en-bh/HT209143").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/appchoices/id894822870").unwrap(),
+Url::parse("https://support.apple.com/safari").unwrap(),
+Url::parse("https://support.apple.com/es-es/safari").unwrap(),
+Url::parse("https://podcasts.apple.com/il/podcast/cpradio/id1491720376").unwrap(),
+Url::parse("https://apps.apple.com/ca/app/thestar-com-iphone/id379481068").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/78-logflare-with-chase-granberry/id1516100616?i=1000545628124").unwrap(),
+Url::parse("https://support.apple.com/en-gb/guide/safari/manage-cookies-and-website-data-sfri11471/mac").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/73-elixir-at-supabase-with-paul-copplestone/id1516100616?i=1000542087899").unwrap(),
+Url::parse("https://developer.apple.com/library/mac/documentation/MobileCoreServices/Reference/UTTypeRef/#//apple_ref/doc/constant_group/UTI_Text_Types").unwrap(),
+Url::parse("https://developer.apple.com/library/prerelease/ios/documentation/Foundation/Reference/NSItemProvider_Class/index.html").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/Foundation/Reference/NSExtensionItem_Class/#//apple_ref/occ/instp/NSExtensionItem/attachments").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/Foundation/Reference/NSExtensionItem_Class").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/Foundation/Reference/NSExtensionContext_Class").unwrap(),
+Url::parse("https://developer.apple.com/library/prerelease/ios/documentation/General/Conceptual/ExtensibilityPG/NotificationCenter.html").unwrap(),
+Url::parse("https://developer.apple.com/videos/play/wwdc2022/110362").unwrap(),
+Url::parse("https://music.apple.com/artist/249646").unwrap(),
+Url::parse("https://developer.apple.com/library/content/documentation/DeveloperTools/Conceptual/MachOTopics/1-Articles/x86_64_code.html").unwrap(),
+Url::parse("https://opensource.apple.com/source/ld64/ld64-123.2.1/src/abstraction/MachOTrie.hpp").unwrap(),
+Url::parse("https://support.apple.com/en-in/guide/safari/sfri11471/mac").unwrap(),
+Url::parse("https://apps.apple.com/app/tmbapp-metro-bus-barcelona/id387847254?pt=216671&ct=web_tmb&mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/it/app/itpc-carabinieri/id858588594?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/tw/app/dramexchange-world-leading/id1080865522?mt=8").unwrap(),
+Url::parse("http://www.apple.com/downloads/macosx/development_tools/svnx.html").unwrap(),
+Url::parse("https://apps.apple.com/ch/app/hostpoint-meet/id1511503042").unwrap(),
+Url::parse("http://www.apple.com/music/store").unwrap(),
+Url::parse("http://support.apple.com/kb/HT4009").unwrap(),
+Url::parse("http://www.apple.com/support/itunes/account").unwrap(),
+Url::parse("http://www.apple.com/support/ipad").unwrap(),
+Url::parse("https://podcasts.apple.com/podcast/id1559261829").unwrap(),
+Url::parse("https://developer.apple.com/library/mac/releasenotes/DeveloperTools/RN-Xcode/Chapters/xc6_release_notes.html#//apple_ref/doc/uid/TP40001051-CH4-SW10").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/unstyled/id1171140955").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/the-cs-ed-podcast/id1486344352").unwrap(),
+Url::parse("https://apps.apple.com/us/app/speechify-text-to-speech-audio/id1209815023").unwrap(),
+Url::parse("https://itunes.apple.com/es/app/marina-dor/id925265167?mt=8").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2020/11/apple-unleashes-m1").unwrap(),
+Url::parse("https://developer.apple.com/news/?id=09032019a").unwrap(),
+Url::parse("https://developer.apple.com/library/archive/documentation/General/Conceptual/ExtensibilityPG/Finder.html").unwrap(),
+Url::parse("https://developer.apple.com/devcenter/download.action?path=/videos/wwdc_2012__hd/session_241__cocoa_interprocess_communication_with_xpc.mov").unwrap(),
+Url::parse("https://opensource.apple.com/source/text_cmds/text_cmds-68/wc/wc.c.auto.html").unwrap(),
+Url::parse("https://support.apple.com/guide/shortcuts-mac/welcome/mac").unwrap(),
+Url::parse("https://developer.apple.com/metal/shader-converter").unwrap(),
+Url::parse("https://support.apple.com/en-ie/HT202074").unwrap(),
+Url::parse("https://apps.apple.com/app/userscripts/id1463298887").unwrap(),
+Url::parse("http://linkmaker.itunes.apple.com").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/smart-author/id1298180300?mt=2").unwrap(),
+Url::parse("https://developer.apple.com/documentation/macos-release-notes/macos-13_4-release-notes").unwrap(),
+Url::parse("https://books.apple.com/book/packaging-for-apple-administrators/id1173928620?mt=11&app=itunes&at=1001lpyX&ct=newsletter").unwrap(),
+Url::parse("https://developer.apple.com/documentation/visionos").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT202491").unwrap(),
+Url::parse("https://geo.itunes.apple.com/us/book/packaging-for-apple-administrators/id1173928620?ls=1&mt=11&at=1001lpyX").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2023/05/app-store-stopped-more-than-2-billion-in-fraudulent-transactions-in-2022").unwrap(),
+Url::parse("https://developer.apple.com/wwdc23/10165").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2023/05/apple-previews-live-speech-personal-voice-and-more-new-accessibility-features").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213544").unwrap(),
+Url::parse("https://developer.apple.com/documentation/Combine").unwrap(),
+Url::parse("https://developer.apple.com/documentation/foundation/userdefaults").unwrap(),
+Url::parse("https://support.apple.com/kb/DL1998").unwrap(),
+Url::parse("https://developer.apple.com/wwdc20/10185").unwrap(),
+Url::parse("https://books.apple.com/book/packaging-for-apple-administrators/id1173928620?mt=11&app=itunes&at=1001lpyX").unwrap(),
+Url::parse("https://developer.apple.com/wwdc21/10095").unwrap(),
+Url::parse("https://developer.apple.com/go/?id=macos-10.15-sdk-rn").unwrap(),
+Url::parse("https://developer.apple.com/documentation/safari-release-notes/safari-16_5-release-notes").unwrap(),
+Url::parse("https://developer.apple.com/documentation/swift/commandline#topics").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT207245").unwrap(),
+Url::parse("https://developer.apple.com/wwdc21/10261").unwrap(),
+Url::parse("https://developer.apple.com/documentation/security/notarizing_macos_software_before_distribution").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT202998").unwrap(),
+Url::parse("https://developer.apple.com/documentation/security/notarizing_macos_software_before_distribution/customizing_the_notarization_workflow").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213327").unwrap(),
+Url::parse("https://developer.apple.com/wwdc23/10053").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213765").unwrap(),
+Url::parse("https://support.apple.com/guide/certifications/welcome/web").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT208891").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT207243").unwrap(),
+Url::parse("https://www.apple.com/ios/ios-17-preview").unwrap(),
+Url::parse("https://developer.apple.com/wwdc23/10002").unwrap(),
+Url::parse("https://geo.itunes.apple.com/us/book/learning-the-bash-shell/id396890728?mt=11&at=1001lpyX").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213760").unwrap(),
+Url::parse("https://developer.apple.com/wwdc23/10263").unwrap(),
+Url::parse("https://developer.apple.com/documentation/SwiftUI").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213250").unwrap(),
+Url::parse("https://support.apple.com/guide/certifications/document-revision-history-apc353b1b736/web").unwrap(),
+Url::parse("https://developer.apple.com/videos/play/wwdc2021/10132").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT207242").unwrap(),
+Url::parse("https://security.apple.com/blog/what-if-we-had-sockpuppet-in-ios16").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT207936").unwrap(),
+Url::parse("https://support.apple.com/guide/deployment/document-revision-history-dep9d814c555/web").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213810").unwrap(),
+Url::parse("https://www.apple.com/ipados/ipados-17-preview").unwrap(),
+Url::parse("https://developer.apple.com/wwdc23/10039").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213812").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213757").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213407").unwrap(),
+Url::parse("https://developer.apple.com/design/human-interface-guidelines/designing-for-visionos").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT207246").unwrap(),
+Url::parse("https://developer.apple.com/documentation/xcode/writing-symbol-documentation-in-your-source-files").unwrap(),
+Url::parse("https://developer.apple.com/wwdc23/10041").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2023/05/apple-launches-new-concert-discovery-features-on-apple-maps-and-apple-music").unwrap(),
+Url::parse("https://developer.apple.com/wwdc23/10040").unwrap(),
+Url::parse("https://developer.apple.com/documentation/foundation/url").unwrap(),
+Url::parse("https://geo.itunes.apple.com/book/property-lists-preferences-profiles-for-apple-administrators/id1213903756?ls=1&mt=11&at=1001lpyX&ct=bookpage").unwrap(),
+Url::parse("https://developer.apple.com/wwdc21/10018").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT207247").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213408").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213758").unwrap(),
+Url::parse("https://developer.apple.com/documentation/xcode-release-notes/xcode-15-release-notes").unwrap(),
+Url::parse("https://books.apple.com/book/terminal-and-shell/id1550617226?mt=11&app=itunes&at=1001lpyX").unwrap(),
+Url::parse("https://developer.apple.com/documentation/technotes/tn3147-migrating-to-the-latest-notarization-tool").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213761").unwrap(),
+Url::parse("https://developer.apple.com/wwdc21/10062").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213813").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213470").unwrap(),
+Url::parse("https://developer.apple.com/documentation/ios-ipados-release-notes/ios-ipados-16_5-release-notes").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT204451").unwrap(),
+Url::parse("https://support.apple.com/guide/tv/whats-new-in-tvos-16-atvb457e13ae/tvos").unwrap(),
+Url::parse("https://www.apple.com/apple-vision-pro").unwrap(),
+Url::parse("https://developer.apple.com/tutorials/swiftui").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213268").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT207244").unwrap(),
+Url::parse("https://developer.apple.com/documentation/swiftui/building-layouts-with-stack-views").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213317").unwrap(),
+Url::parse("https://developer.apple.com/documentation/corefoundation/preferences_utilities").unwrap(),
+Url::parse("https://www.apple.com/watchos/watchos-preview").unwrap(),
+Url::parse("https://developer.apple.com/wwdc23/10120").unwrap(),
+Url::parse("https://developer.apple.com/library/archive/technotes/tn2065/_index.html").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213417").unwrap(),
+Url::parse("https://developer.apple.com/wwdc20/10040").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213759").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213808").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213762").unwrap(),
+Url::parse("https://developer.apple.com/documentation/coreservices/launch_services").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT202999").unwrap(),
+Url::parse("https://developer.apple.com/documentation/appkit/nsworkspace#topics").unwrap(),
+Url::parse("https://books.apple.com/book/moving-to-zsh/id1483591353?mt=11&app=itunes&at=1001lpyX").unwrap(),
+Url::parse("https://support.apple.com/guide/deployment/whats-new-dep950aed53e/web").unwrap(),
+Url::parse("https://support.apple.com/kb/HT213811").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT210060").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT208714").unwrap(),
+Url::parse("https://developer.apple.com/wwdc20/10119").unwrap(),
+Url::parse("https://developer.apple.com/documentation/tvos-release-notes/tvos-16_5-release-notes").unwrap(),
+Url::parse("https://developer.apple.com/documentation/swiftui/appstorage").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213764").unwrap(),
+Url::parse("https://developer.apple.com/bug-reporting").unwrap(),
+Url::parse("https://www.apple.com/macos/sonoma-preview").unwrap(),
+Url::parse("https://developer.apple.com/wwdc23/10254").unwrap(),
+Url::parse("https://books.apple.com/us/book/moving-to-zsh/id1483591353?mt=11&app=itunes&at=1001lpyX").unwrap(),
+Url::parse("https://developer.apple.com/documentation/xcode-release-notes/xcode-14_3_1-release-notes").unwrap(),
+Url::parse("https://support.apple.com/guide/personal-safety/welcome/web").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT201222").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213752").unwrap(),
+Url::parse("https://geo.itunes.apple.com/us/book/bash-cookbook/id396898692?mt=11&at=1001lpyX").unwrap(),
+Url::parse("https://developer.apple.com/wwdc21/10192").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213436").unwrap(),
+Url::parse("https://developer.apple.com/learn/curriculum").unwrap(),
+Url::parse("https://developer.apple.com/wwdc23/102").unwrap(),
+Url::parse("https://geo.itunes.apple.com/nl/book/take-control-mac-command-line-terminal-second-edition/id1234016278?l=en&mt=11&at=1001lpyX").unwrap(),
+Url::parse("https://developer.apple.com/wwdc23/10256").unwrap(),
+Url::parse("https://www.apple.com/newsroom/2023/05/apple-brings-final-cut-pro-and-logic-pro-to-ipad").unwrap(),
+Url::parse("https://geo.itunes.apple.com/book/classic-shell-scripting/id396887958?mt=11&at=1001lpyX").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT201224").unwrap(),
+Url::parse("https://geo.itunes.apple.com/us/book/packaging-for-apple-administrators/id1173928620?ls=1&mt=11&at=1001lpyX&ct=bookpage").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT208050").unwrap(),
+Url::parse("https://books.apple.com/book/moving-to-zsh/id1483591353?mt=11&app=itunes&at=1001lpyX&ct=newsletter").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213809").unwrap(),
+Url::parse("https://developer.apple.com/wwdc23/101").unwrap(),
+Url::parse("https://developer.apple.com/visionos").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT213814").unwrap(),
+Url::parse("https://developer.apple.com/documentation/appkit/nsworkspace").unwrap(),
+Url::parse("https://apps.apple.com/us/app/afends-au/id1568128486").unwrap(),
+Url::parse("https://podcasts.apple.com/ee/podcast/carbon-visibility-in-supply-chains/id1577110956?i=1000597928373").unwrap(),
+Url::parse("https://podcasts.apple.com/ee/podcast/the-what-and-why-of-sustainable-freight/id1577110956?i=1000595798531").unwrap(),
+Url::parse("https://podcasts.apple.com/ee/podcast/how-to-build-a-sustainable-freight-ecosystem/id1577110956?i=1000596983877").unwrap(),
+Url::parse("https://support.apple.com/kb/PH17191").unwrap(),
+Url::parse("https://support.apple.com/es_ES/downloads").unwrap(),
+Url::parse("https://www.apple.com/es/itunes").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/raise-discounted-gift-cards/id922643195?mt=8").unwrap(),
+Url::parse("https://apps.apple.com/us/app/%E6%97%A5%E6%9C%AC%E5%A7%93%E6%B0%8F%E8%AA%9E%E6%BA%90%E8%BE%9E%E5%85%B8-%E3%82%AA%E3%83%95%E3%83%A9%E3%82%A4%E3%83%B3/id1471109324?mt=8").unwrap(),
+Url::parse("https://apps.apple.com/us/app/pocketbook-reader/id805488884").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/the-atlas-obscura-podcast/id1555769970").unwrap(),
+Url::parse("https://apps.apple.com/us/app/microsoft-teams/id1113153706").unwrap(),
+Url::parse("https://apps.apple.com/lt/app/seb-lietuva-m-bankas/id445885936").unwrap(),
+Url::parse("https://apps.apple.com/app/myfootprint-seb/id1589232172").unwrap(),
+Url::parse("https://support.apple.com/en-us/guide/safari/sfri11471/mac").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/m1-finance/id1071915644").unwrap(),
+Url::parse("https://apps.apple.com/us/app/m1-finance/id1071915644").unwrap(),
+Url::parse("https://discussions.apple.com/thread/1107629").unwrap(),
+Url::parse("http://lists.apple.com/archives/Objc-language/2005/Aug/msg00021.html").unwrap(),
+Url::parse("http://lists.apple.com/archives/Objc-language/2005/Aug/threads.html#00019").unwrap(),
+Url::parse("http://developer.apple.com/TOOLS/Xcode").unwrap(),
+Url::parse("https://developer.apple.com/library/content/documentation/Xcode/Reference/xcode_ref-Asset_Catalog_Format/index.html").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/north-star-podcast/id1170914572").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/software-engineering-institute-sei-podcast-series/id566573552?mt=2").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/podcasting-smarter/id566573552?mt=2").unwrap(),
+Url::parse("https://apps.apple.com/us/app/3d%E3%82%B9%E3%82%AD%E3%83%A3%E3%83%8A/id1581786259").unwrap(),
+Url::parse("https://apps.apple.com/app/goodscan-3d/id1600905202").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/voxel-draw/id1084634617").unwrap(),
+Url::parse("https://developer.apple.com/library/archive/documentation/TextFonts/Conceptual/CocoaTextArchitecture/TypoFeatures/TextSystemFeatures.html").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/wikipedia/id324715238").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/graphistania-podcast-for-neo4j-graph-database-community/id975377379").unwrap(),
+Url::parse("https://support.apple.com/zh-cn/guide/iphone/iph10d7bdf26/ios").unwrap(),
+Url::parse("https://apps.apple.com/us/app/wave-invoicing/id881629660").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT212608").unwrap(),
+Url::parse("https://apps.apple.com/us/app/tracker-mood-energy-diary/id1187822249#?platform=messages").unwrap(),
+Url::parse("https://support.apple.com/en-gb/guide/safari/welcome/mac").unwrap(),
+Url::parse("https://www.apple.com/mac-mini").unwrap(),
+Url::parse("https://www.apple.com/macbook-air").unwrap(),
+Url::parse("https://apps.apple.com/us/app/noisli/id862773459").unwrap(),
+Url::parse("http://support.apple.com/downloads").unwrap(),
+Url::parse("http://www.apple.com/support").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/lawyers-guns-money/id1332780055?mt=2").unwrap(),
+Url::parse("https://apps.apple.com/us/app/id1520190857").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/big-ideas-in-app-architecture/id1670998452").unwrap(),
+Url::parse("http://www.apple.com/fr/quicktime/download").unwrap(),
+Url::parse("https://geo.itunes.apple.com/us/app/inquire-by-tamper/id1045268097?mt=8&at=11lmME&ct=tamperio").unwrap(),
+Url::parse("http://www.apple.com/es/privacy/use-of-cookies").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT205280").unwrap(),
+Url::parse("https://itunes.apple.com/app/volumio/id1268256519?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/infinity/id1136944945").unwrap(),
+Url::parse("https://itunes.apple.com/de/institution/universitat-konstanz/id604733564").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/cppcast/id968703120").unwrap(),
+Url::parse("https://itunes.apple.com/ie/app/houses-of-the-oireachtas/id783571697?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/app/payoneer/id663338402").unwrap(),
+Url::parse("https://apps.apple.com/us/app/mayo-clinic/id523220194").unwrap(),
+Url::parse("https://apps.apple.com/de/app/stuttgarter-zeitung-app/id517190273").unwrap(),
+Url::parse("https://apps.apple.com/de/app/stuttgarter-zeitung-de/id397131360").unwrap(),
+Url::parse("https://itunes.apple.com/app/depositphotos/id919098408").unwrap(),
+Url::parse("http://www.apple.com/macosx/lion").unwrap(),
+Url::parse("http://www.apple.com/ipad/#smart-cover").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/seek-by-inaturalist/id1353224144").unwrap(),
+Url::parse("https://www.apple.com/maps").unwrap(),
+Url::parse("https://itunes.apple.com/app/apple-store/id982107779").unwrap(),
+Url::parse("https://developer.apple.com/app-sandboxing").unwrap(),
+Url::parse("https://developer.apple.com/library/mac/documentation/UserExperience/Conceptual/AppleHIGuidelines/IconsImages/IconsImages.html#//apple_ref/doc/uid/20000967-TP6").unwrap(),
+Url::parse("https://developer.apple.com/appstore/mac/resources/approval/guidelines.html").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/the-tim-ferriss-show/id863897795").unwrap(),
+Url::parse("https://apps.apple.com/us/app/jetpack-website-builder/id1565481562").unwrap(),
+Url::parse("https://apps.apple.com/ru/app/fandom-videos-news-reviews/id1230063803").unwrap(),
+Url::parse("https://apps.apple.com/us/app/refinitiv-workspace/id1481442629").unwrap(),
+Url::parse("https://itunes.apple.com/app/aliexpress-for-ipad/id854356557").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/aliexpress/id436672029").unwrap(),
+Url::parse("https://apps.apple.com/jp/app/medaca-%E8%87%AA%E5%88%86%E3%81%AE%E5%81%A5%E5%BA%B7%E3%82%92%E5%8F%8E%E7%B4%8D%E3%81%99%E3%82%8B%E3%82%A2%E3%83%97%E3%83%AA/id1190040349?l=en").unwrap(),
+Url::parse("https://machinelearning.apple.com/updates/apple-scholars-aiml-2022").unwrap(),
+Url::parse("https://apps.apple.com/us/app/microscope-journal/id507267328").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/ardan-labs-podcast/id1539547911").unwrap(),
+Url::parse("https://podcasts.apple.com/gb/podcast/thoughtworks-technology-podcast/id881136697?i=1000584890418").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/adweek-magazine/id570229326?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/edsurge-on-air/id972239500#").unwrap(),
+Url::parse("https://apps.apple.com/de/app/bvg-fahrinfo-plus-berlin/id284971745").unwrap(),
+Url::parse("https://apps.apple.com/de/app/taxi-berlin/id443169338").unwrap(),
+Url::parse("https://apps.apple.com/de/app/going-local-berlin/id964438630").unwrap(),
+Url::parse("https://apps.apple.com/app/wearedevs-world-congress-22/id1625710834").unwrap(),
+Url::parse("https://apps.apple.com/us/app/about-berlin/id1425152714").unwrap(),
+Url::parse("https://geo.itunes.apple.com/us/app/serenity-relaxing-ambient/id1185371817?mt=12&at=1010ltPG").unwrap(),
+Url::parse("http://itunes.apple.com/app/id1238649962").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT210380").unwrap(),
+Url::parse("https://www.apple.com/am/mac/m1").unwrap(),
+Url::parse("https://apps.apple.com/us/app/day-one-2-journal-notes/id1044867788").unwrap(),
+Url::parse("https://apps.apple.com/pt/app/go-i-pace/id1397794566").unwrap(),
+Url::parse("https://itunes.apple.com/gb/podcast/nature-podcast/id81934659?mt=2").unwrap(),
+Url::parse("https://developer.apple.com/fonts/TrueType-Reference-Manual/RM06/Chap6AATIntro.html").unwrap(),
+Url::parse("https://support.apple.com/kb/ph21411?locale=en_US").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/lenovo-3d-catalog/id1164430025?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/app/id1290128888").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/wrike/id890048871?mt=8").unwrap(),
+Url::parse("https://music.apple.com/gb/playlist/bbc-philharmonic-orchestra-of-the-year-2023-nominee/pl.5921732a341d42c99337a1a7cde3f5dd?at=1l3v8Ma&ct=playlist_bbc-philharmonic-orchestra-of-t&itscg=30200&itsct=toolbox_linkbuilder&ls=1&app=music").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=H5FWFAEZ86VNJR9Y25GG&pageLoadId=AqScAGXlLsH2Og2q&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=KGS4XCDCERWHBX904KTF&pageLoadId=BIjEUvwHhteyozzz&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=BG1JH1KFKCF86TW7K07Q&pageLoadId=48P5R1nie5laYcsR&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=XA9CGTJQQQ1532V31779&pageLoadId=FRA8JGqyFpDrcSTk&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=W8PGKFYPV3N1DPJRBSF4&pageLoadId=vhYn8vZlccfbXKXb&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=Y5T2MD9Q99EX901MRJ99&pageLoadId=teMLdIeMi7gtY2D0&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=6QGCKS6Q14B2570A9C6Y&pageLoadId=UadY8Hi2MO0XVEmH&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=20XMEQKV4KM399Z3DJZB&pageLoadId=BN9m4ApVmjfeH44O&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=3JMMYVAR6GKJXFXGDP27&pageLoadId=9SJJwwuJOBhbhzfI&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=RZMTNTNP5ES90TBZSYR8&pageLoadId=KCKIJ7L0Eyu0gPyk&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=TN13W97JDJ647QABA4EX&pageLoadId=q0MJhs7b2apczQAB&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=VKAMXR50EWHMB4JY77AC&pageLoadId=Acc4IyWt1DLl5QfM&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=5N50GSZX2S39GSVMR9PJ&pageLoadId=Zn07nkCtOGonOUfH&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=M5K4BK2KPTB7HA82T3N3&pageLoadId=sDYj4djOnGzMLb7j&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=EJB2JSMXK65J3RYWXPFB&pageLoadId=STrIRrqqustEa93k&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=PZYKXDK1QVNSY5FYGFPR&pageLoadId=tCq5Buyk860Wsvq7&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=J2CN9MEFXKCZJE5CNG1B&pageLoadId=Asy3WiZ6T3IJjEsu&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=ZCVE1TKH3CXEMWANVDXA&pageLoadId=Jaz5nrhH7zBNm33X&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=FEHE4CZJZFY4RKGZPGRS&pageLoadId=HY1aRBlNXutNeFGb&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=1V15C26X8B8C9EVPFRE5&pageLoadId=Lcf8tcXGjBQBFT8r&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=KZP2G549BDRS8DGVZVBN&pageLoadId=CqpUiRnw3DAcgm2V&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=NV4XYEXKB796YX2V0KWK&pageLoadId=6zsVgP3A475l073h&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=WN2BSG3GMWR2FGJTD790&pageLoadId=ELScE1ACkLd2oUS6&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=SSFF13ZDNNGKXC5NK3KE&pageLoadId=PgI0C7FfVLAfdQOi&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=df3314c2-582c-4e54-8eed-d5a4bf97c8ca&pf_rd_r=VYNYGWWXA5TD0JZE3812&pageLoadId=t0yablgDG2JNlBcm&creativeId=9adc166d-b28f-4dd5-bfce-d65bdc6a1a13").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=R1TF8MVA6Z9BHFK6HNSF&pageLoadId=JTgn1d4Qj3BTSPJj&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=KAC02X5CEXSN3T6FJ5W3&pageLoadId=pVNbYTHcSRmy9LsU&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=RMA8JG7DQF12C0ARS2E7&pageLoadId=PEZTCuR49ThJaYNh&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=DFCMWRZ7V68X5FN7JGRM&pageLoadId=1QgdRJO0F04rBxB4&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=165TX149S4EG0GDBJ0M9&pageLoadId=hWqe1UDMwDsii6W7&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=HEVQ06X0PSE73XJJVBFH&pageLoadId=uuXTam0qvCOtsPQF&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=366FZTR6NY1RPVNBENZE&pageLoadId=MkwORBurjRrBD971&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=dbf7b85c-b46a-4a4b-bb2a-c1060910ad41&pf_rd_r=NMMNFBVFCY6B34C0HNXW&pageLoadId=3PdqbgS3EdEUUJSU&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=1R524XYNPHAKM5B77PXD&pageLoadId=0vie0Jfx9N5FAFbL&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=HB0Z73RFR7W8QTYG5G9S&pageLoadId=4ZOnh5ioc3Waw1FS&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=15fe8fa9-606b-4ab6-be26-fffbd045d7bd&pf_rd_r=RPME3P7RF5N9N0FMGS3W&pageLoadId=erNIxgdVCZ96jQN0&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=8DSBE2NQEEEBS04BDYJN&pageLoadId=5RiIY5ACXO41alrU&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=F25TQKQWG9D0FH2Y7FXN&pageLoadId=ocGVVaBWzhoZjmg8&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=WTCRF4CKKXB1X7KPYCVA&pageLoadId=hOhQ65k2vK0d9SIN&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=HBGSB527KB2RZJ1WT5P1&pageLoadId=L2qQrbXVZkyUqr1d&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=AXYTRZA4PA7VNVSZGGGE&pageLoadId=yVy7b49EZPYG1Oij&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=X0EVR7SXRTDK67WNKX64&pageLoadId=PHHJzGzcBpcV3QiQ&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=DJH9066ABTZD7R1NDM2T&pageLoadId=BvMfERP3XO5ytOIi&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=W0NNQX1JYRFCARHHTVDH&pageLoadId=YLi76rAZB9d7uphe&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=34N4KGM51P71DRKQ19P7&pageLoadId=l5IbeFfrkFLvNVf4&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=85RG7R5NG853KQ8TV9HQ&pageLoadId=oR5sHmKaZlpYTxiR&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://itunes.apple.com/jp/app/audiblenoodiobukku/id379693831?mt=8&pf_rd_p=48fb605e-2259-4c40-a135-3b72b4b3a268&pf_rd_r=V57XMRPA5VY6AXP2TFSG&pageLoadId=PmuPbx6SlMFSXPwv&creativeId=8e75c3d7-89e9-4b99-ad55-975ed880f83b").unwrap(),
+Url::parse("https://developer.apple.com/streaming").unwrap(),
+Url::parse("https://developer.apple.com/documentation/http_live_streaming/example_playlists_for_http_live_streaming/creating_a_multivariant_playlist").unwrap(),
+Url::parse("https://itunes.apple.com/es/app/paradores-hoteles-restaurantes/id960788695").unwrap(),
+Url::parse("https://apps.apple.com/us/app/rise-sleep-energy-tracker/id1453884781").unwrap(),
+Url::parse("https://apps.apple.com/us/app/rise-track-sleep-circadian/id1453884781").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/where-we-go-next/id1527944755").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/the-ai-podcast/id1186480811?mt=2&adbsc=social_20161220_68874946&adbid=811257941365882882&adbpl=tw&adbpr=61559439").unwrap(),
+Url::parse("https://developer.apple.com/augmented-reality").unwrap(),
+Url::parse("http://itunes.apple.com/us/app/wikihow-how-to-diy-survival/id309209200?mt=8").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT206481").unwrap(),
+Url::parse("https://developer.apple.com/support/code-signing").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/view_controllers/adding_a_document_browser_to_your_app/presenting_selected_documents").unwrap(),
+Url::parse("https://developer.apple.com/ios/human-interface-guidelines/system-capabilities/quick-look").unwrap(),
+Url::parse("https://developer.apple.com/xcode/swiftui").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uidocumentbrowserviewcontrollerdelegate/2874187-documentbrowser").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uidocumentbrowserviewcontroller").unwrap(),
+Url::parse("https://developer.apple.com/documentation/fileprovider").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619978-contents").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/view_controllers/building_a_document_browser_based_app").unwrap(),
+Url::parse("https://developer.apple.com/library/content/documentation/Miscellaneous/Reference/UTIRef/Articles/System-DeclaredUniformTypeIdentifiers.html").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/airstitch/id645714375?mt=8").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uidocument").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619988-save").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619979-init").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619992-filetype").unwrap(),
+Url::parse("https://developer.apple.com/library/content/documentation/General/Reference/InfoPlistKeyReference/Articles/iPhoneOSKeys.html#//apple_ref/doc/uid/TP40009252-SW37").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619982-documentstate").unwrap(),
+Url::parse("https://www.apple.com/macos/catalina-preview").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/view_controllers/adding_a_document_browser_to_your_app").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619971-load").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uiviewcontroller/1621380-present").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uidocument/1619977-open").unwrap(),
+Url::parse("https://developer.apple.com/library/content/documentation/General/Reference/InfoPlistKeyReference/Articles/iPhoneOSKeys.html#//apple_ref/doc/uid/TP40009252-SW20").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uidocumentbrowserviewcontrollerdelegate").unwrap(),
+Url::parse("https://apps.apple.com/us/app/c-by-ge/id1082828238?ls=1").unwrap(),
+Url::parse("https://music.apple.com/us/artist/dawn-margie-beaton/414795545").unwrap(),
+Url::parse("http://itunes.apple.com/de/app/openseamap/id495210783").unwrap(),
+Url::parse("http://itunes.apple.com/de/app/openseamap/id495210783?mt=8").unwrap(),
+Url::parse("https://apps.apple.com/us/app/3cx-webmeeting/id1039756959").unwrap(),
+Url::parse("https://apps.apple.com/us/app/3cxphone-15-for-3cx-phone/id992045982?ls=1").unwrap(),
+Url::parse("https://apps.apple.com/us/app/3cx/id992045982").unwrap(),
+Url::parse("https://apps.apple.com/us/app/mot%C3%AE/id1046122820?platform=iphone").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/involve-ai-bolstering-customer-success-with-ai-with/id1517181598?i=1000556587495").unwrap(),
+Url::parse("https://apps.apple.com/us/app/work-mobile/id1127865121").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/gopayment/id324389392?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/smart-id/id1188304729?l=et&ls=1&mt=8").unwrap(),
+Url::parse("https://www.apple.com/ios").unwrap(),
+Url::parse("https://support.apple.com/kb/PH17191?locale=es_ES").unwrap(),
+Url::parse("https://itunes.apple.com/app/apple-store/id473512078?pt=356848&at=1l3vpXa&ct=hp&mt=8").unwrap(),
+Url::parse("https://support.apple.com/en-il/guide/safari/sfri11471/mac").unwrap(),
+Url::parse("https://support.apple.com/en-gb/guide/voiceover/vo14111/10/mac/12.0").unwrap(),
+Url::parse("https://support.apple.com/en-gb/guide/iphone/iph3e2e2281/ios").unwrap(),
+Url::parse("https://support.apple.com/en-gb/HT210417").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT206175").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/clustered-conversations-by-weka/id1621362291").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/the-gamesindustry-biz-podcast/id1195829295").unwrap(),
+Url::parse("http://www.apple.com/quicktime/download").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/distributed-with-matt-mullenweg/id1463243282").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/airvisual-real-time-forecast/id1048912974?ls=1&mt=8%27").unwrap(),
+Url::parse("https://itunes.apple.com/gb/app/degiro-beleggen/id1099174456?l=en&mt=8").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/episode-123-dr-martin-weihrauch-smart-in-media-digital/id1490210201?i=1000576089087").unwrap(),
+Url::parse("https://developer.apple.com/documentation/webkit/wkwebview").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/filmstemmer/id1682882422").unwrap(),
+Url::parse("https://apps.apple.com/app/id766443283?mt=8").unwrap(),
+Url::parse("https://apps.apple.com/us/app/tableau-mobile/id434633927").unwrap(),
+Url::parse("https://apps.apple.com/us/app/id1517276406").unwrap(),
+Url::parse("https://apps.apple.com/ch/app/sbb-mobile/id294855237").unwrap(),
+Url::parse("https://apps.apple.com/gb/app/eyeora-vr/id1501679625").unwrap(),
+Url::parse("https://podcasts.apple.com/de/podcast/t3n-catch-up/id1627024991").unwrap(),
+Url::parse("https://podcasts.apple.com/de/podcast/t3n-podcast-das-w%C3%B6chentliche-update-f%C3%BCr-digitale-pioniere/id1115601393").unwrap(),
+Url::parse("https://podcasts.apple.com/de/podcast/t3n-daily/id1643152920").unwrap(),
+Url::parse("https://developer.apple.com/forums/thread/650026").unwrap(),
+Url::parse("https://apps.apple.com/nl/app/wandelnet-nl-wandelroutes/id1463429180").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/it-happened-in-hollywood/id1437795866").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/richard-obrien-the-rocky-horror-picture-show/id1437795866?i=1000617840833").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/between-the-brackets-a-mediawiki-podcast/id1346062023?ls=1").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/the-agile-uprising-podcast/id1163230424?ls=1").unwrap(),
+Url::parse("http://itunes.apple.com/us/podcast/edward-tufte-art-and-science/id297775181").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/code-ish/id1457879893").unwrap(),
+Url::parse("https://apps.apple.com/us/app/google-adsense/id680739529").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/alphabet-scoop/id1359922601?mt=2").unwrap(),
+Url::parse("https://itunes.apple.com/app/apple-store/id626148774?pt=2050088&ct=Modal&mt=8").unwrap(),
+Url::parse("http://opensource.apple.com/source/dyld/dyld-210.2.3/src/dyld_stub_binder.s").unwrap(),
+Url::parse("https://itunes.apple.com/ca/app/bc-lions-exp/id1263298513?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/es/app/telegram-messenger/id686449807?mt=8").unwrap(),
+Url::parse("https://developer.apple.com/documentation/security/certificate_key_and_trust_services/trust?language=objc").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT210176").unwrap(),
+Url::parse("https://developer.apple.com/documentation/security/secure_transport").unwrap(),
+Url::parse("https://developer.apple.com/library/archive/documentation/Darwin/Conceptual/KernelProgramming/BSD/BSD.html#//apple_ref/doc/uid/TP30000905-CH214-TPXREF101").unwrap(),
+Url::parse("https://developer.apple.com/library/archive/documentation/System/Conceptual/ManPages_iPhoneOS/man2/getsockopt.2.html").unwrap(),
+Url::parse("https://apps.apple.com/us/app/wallmine-stocks-bitcoin/id1458107499").unwrap(),
+Url::parse("https://apps.apple.com/us/app/dfds-ferries-cruises/id1447470946").unwrap(),
+Url::parse("https://itunes.apple.com/in/app/valley-air-district/id526195423").unwrap(),
+Url::parse("https://support.apple.com/en-gb/HT211094").unwrap(),
+Url::parse("https://itunes.apple.com/app/apple-store/id954157737?pt=49113808&ct=georgegarside.com%2Fapps&mt=8&at=1001lwpF").unwrap(),
+Url::parse("https://itunes.apple.com/app/apple-store/id989353027?pt=49113808&ct=georgegarside.com%2Fapps&mt=8&at=1001lwpF").unwrap(),
+Url::parse("http://www.apple.com/trailers/fox/thehappening").unwrap(),
+Url::parse("http://itunes.apple.com/gb/podcast/british-library-podcasts/id174455610?ign-mpt=uo=4").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/apple-store/id535886823?pt").unwrap(),
+Url::parse("https://support.apple.com/it-it/guide/safari/sfri11471/mac").unwrap(),
+Url::parse("https://support.apple.com/en-ca/guide/safari/sfri11471/mac").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/foodist-with-darya-rose-ph.d/id1090954776?mt=2").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/foodist-with-darya-rose-ph-d/id1090954776").unwrap(),
+Url::parse("https://developer.apple.com/documentation/app_clips/creating_an_app_clip").unwrap(),
+Url::parse("https://developer.apple.com/documentation/bundleresources/information_property_list/nslocationwheninuseusagedescription").unwrap(),
+Url::parse("https://developer.apple.com/design/human-interface-guidelines/ios/system-capabilities/notifications").unwrap(),
+Url::parse("https://developer.apple.com/documentation/corelocation/choosing_the_authorization_level_for_location_services").unwrap(),
+Url::parse("https://developer.apple.com/documentation/usernotifications/unnotificationserviceextension").unwrap(),
+Url::parse("https://developer.apple.com/documentation/uikit/uiapplicationdelegate/1623013-application").unwrap(),
+Url::parse("https://developer.apple.com/documentation/app_clips/enabling_notifications_in_app_clips").unwrap(),
+Url::parse("https://developer.apple.com/library/archive/qa/qa1633/_index.html").unwrap(),
+Url::parse("https://developer.apple.com/documentation/bundleresources/information_property_list/nslocationalwaysusagedescription").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT203609").unwrap(),
+Url::parse("https://developer.apple.com/documentation/usernotifications/asking_permission_to_use_notifications").unwrap(),
+Url::parse("https://itunes.apple.com/ch/app/india-air-quality/id988591878?l=en&mt=8").unwrap(),
+Url::parse("https://developer.apple.com/documentation/widgetkit/timelineprovider").unwrap(),
+Url::parse("https://developer.apple.com/videos/play/wwdc2020/10034").unwrap(),
+Url::parse("https://developer.apple.com/documentation/widgetkit/creating-a-widget-extension").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT208309").unwrap(),
+Url::parse("https://support.apple.com/guide/shortcuts/add-import-questions-before-sharing-apdf330fd3a0/ios").unwrap(),
+Url::parse("https://support.apple.com/guide/shortcuts/welcome/ios").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/google-photos-store-search/id962194608?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/app/id853371601").unwrap(),
+Url::parse("https://developer.apple.com/technologies/ios").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/google-translate/id414706506?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/google-slides/id879478102?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/youtube/id544007664?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/google-docs/id842842640?mt=8").unwrap(),
+Url::parse("http://developer.apple.com/iphone/index.action").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/google-sheets/id842849113?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/app/idid909319292").unwrap(),
+Url::parse("https://itunes.apple.com/app/id905060486").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/google-play-music/id691797987?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/google-calendar/id909319292?mt=8").unwrap(),
+Url::parse("http://developer.apple.com/opensource").unwrap(),
+Url::parse("https://developer.apple.com/library/ios/documentation/CoreBluetooth/Reference/CBCentralManager_Class/index.html#//apple_ref/occ/instm/CBCentralManager/cancelPeripheralConnection:").unwrap(),
+Url::parse("https://developer.apple.com/library/prerelease/ios/documentation/CoreBluetooth/Reference/CBCentralManager_Class/index.html#//apple_ref/doc/constant_group/Central_Manager_Initialization_Options").unwrap(),
+Url::parse("https://developer.apple.com/library/prerelease/ios/documentation/CoreBluetooth/Reference/CBPeripheral_Class/index.html#//apple_ref/c/tdef/CBCharacteristicWriteType").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/brian-collins/id328074695?i=1000384718204&mt=2").unwrap(),
+Url::parse("https://apps.apple.com/us/app/syncfusion-succinctly-series/id1336430099").unwrap(),
+Url::parse("https://www.apple.com/uk/swift").unwrap(),
+Url::parse("https://apps.apple.com/nz/app/1-news/id329786380").unwrap(),
+Url::parse("https://apps.apple.com/app/apple-store/id1081530898").unwrap(),
+Url::parse("https://apps.apple.com/us/app/the-new-yorker-today/id1081530898").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT202039").unwrap(),
+Url::parse("https://apps.apple.com/us/app/flipboard-latest-stories").unwrap(),
+Url::parse("https://apps.apple.com/us/app/the-new-yorker/id1081530898").unwrap(),
+Url::parse("https://apps.apple.com/be/app/go-i-pace/id1397794566?l=fr").unwrap(),
+Url::parse("https://apps.apple.com/be/app/go-i-pace/id1397794566?l=nl").unwrap(),
+Url::parse("https://apps.apple.com/app/apple-store/id1241040030?pt=64957&ct=blizzard-global-footer&mt=8").unwrap(),
+Url::parse("http://www.apple.com/macosx").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/mht-to-webarchive/id838885505?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/css-tidy/id700229660?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/tcl-runner/id701474998?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/python-lint/id891162632?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/python-runner/id689176796?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/php-lint/id878546858?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/web-proxy/id721103033?mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/huo-che-ding-piao-zhu-shou/id494486644?ls=1&mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/socks-proxy/id1116965059?mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/set-file-icon/id787613939?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/copy-as-text/id869232291?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/html-tidy/id700229601?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/php-runner/id686855602?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/chinese-conversion/id720125982?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/no-trash-delete-files-without/id777213066?ls=1&mt=12").unwrap(),
+Url::parse("https://apps.apple.com/app/new-file-menu/id1064959555?l=en&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/webarchive-to-mht/id838885411?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/ip-messenger/id875939790?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/chmplus-chm-reader/id588628901?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/jslint/id757770685?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/simple-svn-server/id1040404148?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/epub-reader/id591299853?ls=1&mt=12").unwrap(),
+Url::parse("https://apps.apple.com/cn/story/id1520078797?l=en").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/copy-as-html/id868049821?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/simple-hg-server/id1040403596?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/svn-status/id842480930?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/extractor-extract-rar-zip/id747644886?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/css-beautifier/id757811119?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/monitor-test/id904107000?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/favicon-creator/id787259510?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/simple-git-server/id1040403936?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/mht-viewer/id622609427?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/app/context-menu/id1236813619?l=en&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/copy-file-path/id777213048?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/hg-status/id993527882?mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/shell-checker/id892605935?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/paste-as-file/id825675368?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/archive-viewer/id598123939?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/dock-icons/id788702540?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/json-lint/id894042665?ls=1&mt=12").unwrap(),
+Url::parse("https://support.apple.com/kb/ht4094").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/lua-runner/id689181261?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/icon-extractor/id803083078?ls=1&mt=12").unwrap(),
+Url::parse("https://apps.apple.com/app/context-menu/id1236813619?l=en&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/snailfossil/id1226366997?ls=1&mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/webarchive-to-html/id844587292?ls=1&mt=12").unwrap(),
+Url::parse("https://apps.apple.com/app/git-credential-manager/id1575689398?l=en&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/node-runner/id689175950?ls=1&mt=12").unwrap(),
+Url::parse("https://apps.apple.com/app/java-runner/id689184215").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/perl-runner/id689178925?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/icon-converter/id803070906?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/html5-tidy/id700229688?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/startup-time/id875911449?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/app/chmplus-pro-chm-reader/id441521818?mt=8").unwrap(),
+Url::parse("https://apps.apple.com/app/filedrop-quick-file-transfer/id1605772776?l=en&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/app/webdav-server/id1185282799?l=en&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/git-status/id842481797?mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/new-file-here/id777221999?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/app/new-file-menu-free/id1066302071?l=en&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/chm-to-html/id869362084?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/share-clipboard/id619587463?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/ipagerank-pagerank-checker/id433671401?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/ruby-runner/id689179916?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/js-runner/id689183220?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/js-beautifier/id700229527?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/source-code-counter/id889599066?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/mht-to-html/id844591996?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/clipboard-viewer/id870366108?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/qr-clipboard/id869183896?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/chm-lite/id443454952?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/cn/artist/langui.net/id433671404?l=en&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/app/new-file-menu/id1064959555?l=en&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/snailhg/id1006394020?mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/app/chmplus-lite-chm-reader/id443454952?mt=8").unwrap(),
+Url::parse("https://apps.apple.com/app/isvn/id861041982?ls=1").unwrap(),
+Url::parse("https://apps.apple.com/app/ftp-mounter/id1437045769?l=en&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/hg-server/id1034692383?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/git-server/id959332098?ls=1&mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/cn/app/huo-che-ding-piao-zhu-shou/id494486644?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/app/chmplus-chm-reader/id441521818?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/network-speed-status/id875176096?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/chmplus-free-chm-reader/id554596300?ls=1&mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/app/zipmounter/id1315374401?l=en&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/smartcopy/id884734329?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/svn-server/id685191052?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/sftp-server/id1300094971?ls=1&mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/css-lint/id757865011?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/new-file-menu/id1064959555?ls=1&mt=12").unwrap(),
+Url::parse("https://itunes.apple.com/nl/app/logius/id1167926346?ls=1&mt=8").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/e356-jeff-seibert-co-founder-at-digits/id1442272943?i=1000567828292").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/episode-2-fintech-what-the-heck-digits/id1578932483?i=1000568891111").unwrap(),
+Url::parse("https://podcasts.apple.com/us/podcast/digitalconomics-podcast/id1611260822").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/ticketmaster/id500003565?mt=8").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/bitfinex/id1436383182?ls=1&mt=8").unwrap(),
+Url::parse("http://support.apple.com/kb/PH17191?viewlocale=fr_FR").unwrap(),
+Url::parse("https://support.apple.com/ja-jp/HT202074").unwrap(),
+Url::parse("https://apps.apple.com/app/apple-store/id1625333066?pt=125060841&ct=website-main&mt=8").unwrap(),
+Url::parse("https://apps.apple.com/app/id1594445686").unwrap(),
+Url::parse("https://itunes.apple.com/app/id911410294").unwrap(),
+Url::parse("https://apps.apple.com/app/realplayer-mobile/id1578950397").unwrap(),
+Url::parse("https://apps.apple.com/us/app/trnio-plus-3d-scanner/id1546160586").unwrap(),
+Url::parse("https://apps.apple.com/us/app/trnio-plus-3d-scanner/id1546160586?itsct=apps_box_badge&itscg=30200").unwrap(),
+Url::parse("https://apps.apple.com/pt/app/3cx/id992045982").unwrap(),
+Url::parse("https://itunes.apple.com/gb/podcast/art-matters/id1343288276?mt=2").unwrap(),
+Url::parse("https://itunes.apple.com/cn/podcast/%E6%9E%81%E5%AE%A2%E7%94%B5%E5%8F%B0/id914427651?mt=2").unwrap(),
+Url::parse("https://support.apple.com/en-us/HT201859").unwrap(),
+Url::parse("https://itunes.apple.com/us/podcast/the-big-picture/id1439252196?mt=2").unwrap(),
+Url::parse("https://itunes.apple.com/us/app/uberfit-runner/id1033916715?mt=8&at=10lwCW&uo=8&at=10lwCW").unwrap(),
+Url::parse("https://www.apple.com/apple-music").unwrap(),
+Url::parse("https://www.apple.com/legal/internet-services/itunes/us/terms.html").unwrap(),
];
// pre-initialize domain prefix list
diff --git a/core/src/alice/generate.rs b/core/src/alice/generate.rs
index f82547a4..75f4db58 100644
--- a/core/src/alice/generate.rs
+++ b/core/src/alice/generate.rs
@@ -67,7 +67,7 @@ impl AliceTokenGenerator {
max_new_tokens: Option<usize>,
) -> Result<Self> {
if tokens.is_empty() {
- return Err(Error::EmptyInput);
+ return Err(Error::EmptyInput.into());
}
// this must be 2, otherwise `.front()` and `.back()` calls are wrong throughout the generation code
diff --git a/core/src/alice/mod.rs b/core/src/alice/mod.rs
index 60ee2612..4299ac24 100644
--- a/core/src/alice/mod.rs
+++ b/core/src/alice/mod.rs
@@ -35,9 +35,11 @@ use aes_gcm::{
aead::{Aead, OsRng},
AeadCore, Aes256Gcm, Key, KeyInit, Nonce,
};
+use anyhow::anyhow;
use flate2::{bufread::GzDecoder, write::GzEncoder, Compression};
use half::bf16;
use tch::Tensor;
+use url::Url;
use crate::{
api::search::ApiSearchQuery,
@@ -47,8 +49,7 @@ use crate::{
llm_utils::ClonableTensor,
search_prettifier::DisplayedWebpage,
searcher::{SearchResult, WebsitesResult},
- summarizer::{self, ExtractiveSummarizer},
- webpage::Url,
+ summarizer::ExtractiveSummarizer,
};
use self::{
@@ -69,58 +70,22 @@ Alice: <answer><|endoftext|>"#;
pub mod generate;
mod raw_model;
-type Result<T> = std::result::Result<T, Error>;
+type Result<T> = std::result::Result<T, anyhow::Error>;
#[derive(thiserror::Error, Debug)]
pub enum Error {
- #[error("IO error: {0}")]
- Io(#[from] std::io::Error),
-
- #[error("Torch error: {0}")]
- Torch(#[from] tch::TchError),
-
- #[error("SafeTensors error: {0}")]
- SafeTensors(#[from] safetensors::SafeTensorError),
-
- #[error("Tokenizers error: {0}")]
- Tokenizers(#[from] tokenizers::Error),
-
#[error("Empty input")]
EmptyInput,
- #[error("Reqwest error: {0}")]
- Reqwest(#[from] reqwest::Error),
-
#[error("Unexpected search result")]
UnexpectedSearchResult,
- #[error("Summarizer: {0}")]
- Summarizer(#[from] summarizer::Error),
-
- #[error("Bincode: {0}")]
- Bincode(#[from] bincode::Error),
-
- #[error("Base64: {0}")]
- Base64(#[from] base64::DecodeError),
-
#[error("Failed to decrypt")]
DecryptionFailed,
- #[error("Cluster")]
- Cluster(#[from] crate::distributed::cluster::Error),
-
#[error("Unexpected completion")]
UnexpectedCompletion,
- #[error("Event source cannot clone")]
- EventSourceCannotClone(#[from] reqwest_eventsource::CannotCloneRequestError),
-
- #[error("Event source error")]
- EventSource(#[from] reqwest_eventsource::Error),
-
- #[error("Serde JSON error")]
- SerdeJson(#[from] serde_json::Error),
-
#[error("Last message should be from user")]
LastMessageNotUser,
}
@@ -136,13 +101,13 @@ pub struct SimplifiedWebsite {
impl SimplifiedWebsite {
fn new(webpage: DisplayedWebpage, query: &str, summarizer: &ExtractiveSummarizer) -> Self {
let text = summarizer.summarize(query, &webpage.body);
- let url = Url::from(webpage.url.to_string());
+ let url = Url::parse(&webpage.url).unwrap();
Self {
title: webpage.title,
text,
- site: url.site().to_string(),
- url: url.full(),
+ site: url.host_str().unwrap_or_default().to_string(),
+ url: url.to_string(),
}
}
}
@@ -213,7 +178,7 @@ impl Searcher {
match res {
SearchResult::Websites(res) => Ok(res),
- SearchResult::Bang(_) => Err(Error::UnexpectedSearchResult),
+ SearchResult::Bang(_) => Err(Error::UnexpectedSearchResult.into()),
}
}
@@ -404,13 +369,16 @@ pub struct Tokenizer {
impl Tokenizer {
pub fn open<P: AsRef<Path>>(path: P) -> Result<Tokenizer> {
- let tokenizer = tokenizers::Tokenizer::from_file(path)?;
+ let tokenizer = tokenizers::Tokenizer::from_file(path).map_err(|e| anyhow!(e))?;
Ok(Self { tokenizer })
}
pub fn encode(&self, input: String) -> Result<Vec<i64>> {
- let encoding = self.tokenizer.encode(input, false)?;
+ let encoding = self
+ .tokenizer
+ .encode(input, false)
+ .map_err(|e| anyhow!(e))?;
let ids = encoding
.get_ids()
@@ -424,7 +392,10 @@ impl Tokenizer {
pub fn decode(&self, tokens: &[i64]) -> Result<String> {
let tokens = tokens.iter().map(|&id| id as u32).collect::<Vec<_>>();
- let output = self.tokenizer.decode(tokens, true)?;
+ let output = self
+ .tokenizer
+ .decode(tokens, true)
+ .map_err(|e| anyhow!(e))?;
Ok(output)
}
diff --git a/core/src/alice/raw_model.rs b/core/src/alice/raw_model.rs
index d62d5c1d..41511229 100644
--- a/core/src/alice/raw_model.rs
+++ b/core/src/alice/raw_model.rs
@@ -15,7 +15,7 @@ use std::{fs::OpenOptions, path::Path};
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::alice::{Error, Result};
+use crate::alice::Result;
use safetensors::SafeTensors;
use tch::{
nn::{embedding, layer_norm, Embedding, LayerNorm, LayerNormConfig, ModuleT, VarStore},
@@ -529,12 +529,9 @@ impl RawModel {
.read(true)
.write(true)
.create(true)
- .open(path)
- .map_err(Error::Io)?;
+ .open(path)?;
- memmap2::MmapOptions::new()
- .map_mut(&file)
- .map_err(Error::Io)?
+ memmap2::MmapOptions::new().map_mut(&file)?
};
let weights = SafeTensors::deserialize(&mmap)?;
diff --git a/core/src/api/alice.rs b/core/src/api/alice.rs
index 36090134..7d8dbe62 100644
--- a/core/src/api/alice.rs
+++ b/core/src/api/alice.rs
@@ -101,7 +101,7 @@ impl EncodedSavedState {
Self(saved_state)
}
- fn decode(self) -> Result<SavedState, crate::alice::Error> {
+ fn decode(self) -> Result<SavedState, anyhow::Error> {
let saved_state = base64::decode(self.0)?;
let saved_state = bincode::deserialize(&saved_state)?;
Ok(saved_state)
diff --git a/core/src/api/improvement.rs b/core/src/api/improvement.rs
index 71cbbd7b..6a831c91 100644
--- a/core/src/api/improvement.rs
+++ b/core/src/api/improvement.rs
@@ -19,6 +19,7 @@ use std::sync::Arc;
use askama::Template;
use axum::{extract, response::IntoResponse};
use serde::Deserialize;
+use url::Url;
use uuid::Uuid;
use crate::improvement::{AliceMessage, ImprovementEvent, StoredQuery};
@@ -50,12 +51,15 @@ pub async fn click(
}
}
-impl From<StoreParams> for StoredQuery {
- fn from(params: StoreParams) -> Self {
- StoredQuery::new(
- params.query,
- params.urls.into_iter().map(|s| s.into()).collect(),
- )
+impl TryFrom<StoreParams> for StoredQuery {
+ type Error = anyhow::Error;
+ fn try_from(params: StoreParams) -> Result<Self, Self::Error> {
+ let mut urls = Vec::new();
+ for url in params.urls {
+ urls.push(Url::parse(&url)?);
+ }
+
+ Ok(StoredQuery::new(params.query, urls))
}
}
@@ -65,13 +69,15 @@ pub async fn store(
extract::Json(params): extract::Json<StoreParams>,
) -> impl IntoResponse {
match state.improvement_queue.as_ref() {
- Some(q) => {
- let query: StoredQuery = params.into();
- let qid = *query.qid();
- q.lock().await.push(ImprovementEvent::StoreQuery(query));
-
- qid.to_string()
- }
+ Some(q) => match StoredQuery::try_from(params) {
+ Ok(query) => {
+ let qid = *query.qid();
+ q.lock().await.push(ImprovementEvent::StoreQuery(query));
+
+ qid.to_string()
+ }
+ Err(_) => String::new(),
+ },
None => String::new(),
}
}
diff --git a/core/src/api/search.rs b/core/src/api/search.rs
index 60f31e90..24085ff7 100644
--- a/core/src/api/search.rs
+++ b/core/src/api/search.rs
@@ -30,7 +30,6 @@ use crate::{
searcher::{self, SearchQuery, SearchResult, WebsitesResult, NUM_RESULTS_PER_PAGE},
webpage::region::{Region, ALL_REGIONS},
widgets::Widget,
- Error,
};
use super::{
@@ -199,16 +198,16 @@ pub async fn route(
Ok(HtmlTemplate(template).into_response())
}
SearchResult::Bang(result) => {
- Ok(Redirect::to(&result.redirect_to.full()).into_response())
+ Ok(Redirect::to(result.redirect_to.as_str()).into_response())
+ }
+ },
+ Err(err) => match err.downcast_ref() {
+ Some(searcher::distributed::Error::EmptyQuery) => Ok(Redirect::to("/").into_response()),
+ _ => {
+ tracing::error!("{:?}", err);
+ Err(StatusCode::INTERNAL_SERVER_ERROR)
}
},
- Err(Error::DistributedSearcher(searcher::distributed::Error::EmptyQuery)) => {
- Ok(Redirect::to("/").into_response())
- }
- Err(err) => {
- tracing::error!("{:?}", err);
- Err(StatusCode::INTERNAL_SERVER_ERROR)
- }
}
}
@@ -281,7 +280,7 @@ pub struct ApiSearchQuery {
}
impl TryFrom<ApiSearchQuery> for SearchQuery {
- type Error = crate::Error;
+ type Error = anyhow::Error;
fn try_from(api: ApiSearchQuery) -> Result<Self, Self::Error> {
let optic = if let Some(optic) = &api.optic {
@@ -310,7 +309,7 @@ impl TryFrom<ApiSearchQuery> for SearchQuery {
#[serde(tag = "@type", rename_all = "camelCase")]
pub enum ApiSearchResult {
Websites(WebsitesResult),
- Bang(BangHit),
+ Bang(Box<BangHit>),
}
impl From<SearchResult> for ApiSearchResult {
@@ -348,14 +347,17 @@ pub async fn api(
Ok(Json(result).into_response())
}
}
- Err(Error::DistributedSearcher(searcher::distributed::Error::EmptyQuery)) => {
- Ok(searcher::distributed::Error::EmptyQuery
- .to_string()
- .into_response())
- }
- Err(err) => {
- tracing::error!("{:?}", err);
- Err(StatusCode::INTERNAL_SERVER_ERROR)
- }
+
+ Err(err) => match err.downcast_ref() {
+ Some(searcher::distributed::Error::EmptyQuery) => {
+ Ok(searcher::distributed::Error::EmptyQuery
+ .to_string()
+ .into_response())
+ }
+ _ => {
+ tracing::error!("{:?}", err);
+ Err(StatusCode::INTERNAL_SERVER_ERROR)
+ }
+ },
}
}
diff --git a/core/src/bangs.rs b/core/src/bangs.rs
index e79f440d..3844c500 100644
--- a/core/src/bangs.rs
+++ b/core/src/bangs.rs
@@ -18,8 +18,9 @@ use std::{collections::HashMap, fs, path::Path};
use itertools::intersperse;
use serde::{Deserialize, Serialize};
+use url::Url;
-use crate::{query::parser::Term, webpage::Url};
+use crate::query::parser::Term;
pub const BANG_PREFIX: char = '!';
@@ -84,29 +85,32 @@ impl Bangs {
}
}) {
if let Some(bang) = self.bangs.get(possible_bang) {
+ let mut url = bang.url.replace(
+ "{{{s}}}",
+ intersperse(
+ terms
+ .iter()
+ .filter(|term| {
+ if let Term::PossibleBang(bang) = term.as_ref() {
+ bang != possible_bang
+ } else {
+ true
+ }
+ })
+ .map(|term| term.as_ref().to_string()),
+ " ".to_string(),
+ )
+ .collect::<String>()
+ .as_str(),
+ );
+
+ if !url.contains("://") {
+ url = "http://".to_string() + url.as_str();
+ }
+
return Some(BangHit {
bang: bang.clone(),
- redirect_to: bang
- .url
- .replace(
- "{{{s}}}",
- intersperse(
- terms
- .iter()
- .filter(|term| {
- if let Term::PossibleBang(bang) = term.as_ref() {
- bang != possible_bang
- } else {
- true
- }
- })
- .map(|term| term.as_ref().to_string()),
- " ".to_string(),
- )
- .collect::<String>()
- .as_str(),
- )
- .into(),
+ redirect_to: Url::parse(url.as_str()).unwrap(),
});
}
}
@@ -151,9 +155,8 @@ mod tests {
tag: "ty".to_string(),
url: "https://www.youtube.com/results?search_query={{{s}}}".to_string()
},
- redirect_to: "https://www.youtube.com/results?search_query=bangs"
- .to_string()
- .into()
+ redirect_to: Url::parse("https://www.youtube.com/results?search_query=bangs")
+ .unwrap()
})
);
}
diff --git a/core/src/crawler/coordinator.rs b/core/src/crawler/coordinator.rs
index 779f3ed8..7adf75ac 100644
--- a/core/src/crawler/coordinator.rs
+++ b/core/src/crawler/coordinator.rs
@@ -14,7 +14,9 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <https://www.gnu.org/licenses/>.
-use crate::{call_counter::CallCounter, webpage::Url};
+use url::Url;
+
+use crate::call_counter::CallCounter;
use super::{
crawl_db::{CrawlDb, DomainStatus},
@@ -45,10 +47,13 @@ impl CrawlCoordinator {
seed_urls: Vec<String>,
) -> Result<Self> {
let mut db = CrawlDb::open(crawldb_folder)?;
+ let mut parsed_seed_urls = Vec::new();
- let seed_urls = seed_urls.into_iter().map(Url::from).collect::<Vec<_>>();
+ for url in seed_urls {
+ parsed_seed_urls.push(Url::parse(&url)?);
+ }
- db.insert_seed_urls(&seed_urls)?;
+ db.insert_seed_urls(&parsed_seed_urls)?;
Ok(Self {
db: Mutex::new(db),
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index f0c6f690..ac410a5e 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -26,8 +26,7 @@ use itertools::Itertools;
use lru::LruCache;
use rand::Rng;
use rocksdb::BlockBasedOptions;
-
-use crate::webpage::Url;
+use url::Url;
use super::{Domain, Job, JobResponse, Result, UrlResponse};
@@ -398,7 +397,7 @@ impl CrawlDb {
for res in responses {
for url in &res.discovered_urls {
- let domain: Domain = url.domain().to_string().into();
+ let domain = Domain::from(url);
let different_domain = res.domain != domain;
domains.entry(domain).or_default().push(UrlToInsert {
@@ -457,7 +456,7 @@ impl CrawlDb {
for url_response in &res.url_responses {
match url_response {
UrlResponse::Success { url } => {
- let domain: Domain = url.domain().to_string().into();
+ let domain = Domain::from(url);
url_responses
.entry(domain)
.or_default()
@@ -467,14 +466,14 @@ impl CrawlDb {
url,
status_code: _,
} => {
- let domain: Domain = url.domain().to_string().into();
+ let domain = Domain::from(url);
url_responses
.entry(domain)
.or_default()
.push(url_response.clone());
}
UrlResponse::Redirected { url, new_url: _ } => {
- let domain: Domain = url.domain().to_string().into();
+ let domain = Domain::from(url);
url_responses
.entry(domain)
.or_default()
diff --git a/core/src/crawler/mod.rs b/core/src/crawler/mod.rs
index b220238a..06db7eba 100644
--- a/core/src/crawler/mod.rs
+++ b/core/src/crawler/mod.rs
@@ -21,13 +21,11 @@ use std::{
};
use tokio::sync::Mutex;
+use url::Url;
-use crate::{config::CrawlerConfig, webpage::Url};
+use crate::config::CrawlerConfig;
-use self::{
- warc_writer::{WarcWriter, WarcWriterMessage},
- worker::Worker,
-};
+use self::{warc_writer::WarcWriter, worker::Worker};
pub mod coordinator;
pub mod crawl_db;
@@ -39,12 +37,6 @@ pub use coordinator::CrawlCoordinator;
#[derive(thiserror::Error, Debug)]
pub enum Error {
- #[error("reqwest error: {0}")]
- Reqwest(#[from] reqwest::Error),
-
- #[error("chrono out-of-range error: {0}")]
- Chrono(#[from] chrono::OutOfRangeError),
-
#[error("invalid content type: {0}")]
InvalidContentType(String),
@@ -56,24 +48,9 @@ pub enum Error {
#[error("invalid politeness factor")]
InvalidPolitenessFactor,
-
- #[error("addr parse error: {0}")]
- AddrParse(#[from] std::net::AddrParseError),
-
- #[error("channel: {0}")]
- SendError(#[from] tokio::sync::mpsc::error::SendError<WarcWriterMessage>),
-
- #[error("bincode: {0}")]
- Bincode(#[from] bincode::Error),
-
- #[error("rocksdb: {0}")]
- Rocksdb(#[from] rocksdb::Error),
-
- #[error("io: {0}")]
- Io(#[from] std::io::Error),
}
-type Result<T> = std::result::Result<T, Error>;
+type Result<T> = std::result::Result<T, anyhow::Error>;
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
struct Site(String);
@@ -83,7 +60,7 @@ pub struct Domain(String);
impl From<&Url> for Domain {
fn from(url: &Url) -> Self {
- Self(url.domain().to_string())
+ Self(url.domain().unwrap_or_default().to_string())
}
}
diff --git a/core/src/crawler/robots_txt.rs b/core/src/crawler/robots_txt.rs
index 99d0a8dd..69e769cb 100644
--- a/core/src/crawler/robots_txt.rs
+++ b/core/src/crawler/robots_txt.rs
@@ -20,7 +20,7 @@ use robotstxt_with_cache::matcher::{
CachingRobotsMatcher, LongestMatchRobotsMatchStrategy, RobotsMatcher,
};
-use crate::webpage::Url;
+use url::Url;
use super::{Error, Result, Site};
@@ -41,7 +41,7 @@ impl RobotsTxtManager {
match self.get_mut(url).await {
Ok(Some(robots_txt)) => robots_txt
.matcher
- .one_agent_allowed_by_robots(user_agent, &url.full()),
+ .one_agent_allowed_by_robots(user_agent, url.as_str()),
_ => true,
}
}
@@ -54,32 +54,36 @@ impl RobotsTxtManager {
.await?;
if res.status() != reqwest::StatusCode::OK {
- return Err(Error::FetchFailed(res.status()));
+ return Err(Error::FetchFailed(res.status()).into());
}
let body = res.text().await?;
match panic::catch_unwind(|| RobotsTxt::new(body)) {
- Ok(r) => r,
- Err(_) => Err(Error::FetchFailed(reqwest::StatusCode::IM_A_TEAPOT)),
+ Ok(r) => Ok(r),
+ Err(_) => Err(Error::FetchFailed(reqwest::StatusCode::IM_A_TEAPOT).into()),
}
}
async fn get_mut(&mut self, url: &Url) -> Result<Option<&mut RobotsTxt>> {
- let site = Site(url.site().to_string());
+ let site = Site(url.host_str().unwrap_or_default().to_string());
if self.cache.get(&site).is_none() {
match self.fetch_robots_txt(&site).await {
Ok(robots_txt) => {
self.cache.insert(site.clone(), Some(robots_txt));
}
- Err(Error::FetchFailed(status)) if status == reqwest::StatusCode::NOT_FOUND => {
- self.cache.insert(site.clone(), None);
- }
- Err(err) => {
- self.cache.insert(site.clone(), None);
- tracing::warn!("failed to fetch robots.txt for {}: {}", site.0, err);
- }
+ Err(err) => match err.downcast_ref() {
+ Some(Error::FetchFailed(status))
+ if *status == reqwest::StatusCode::IM_A_TEAPOT =>
+ {
+ self.cache.insert(site.clone(), None);
+ }
+ _ => {
+ self.cache.insert(site.clone(), None);
+ tracing::warn!("failed to fetch robots.txt for {}: {}", site.0, err);
+ }
+ },
}
}
@@ -103,7 +107,7 @@ struct RobotsTxt {
}
impl RobotsTxt {
- fn new(body: String) -> Result<Self> {
+ fn new(body: String) -> Self {
let mut matcher = CachingRobotsMatcher::new(RobotsMatcher::default());
matcher.parse(&body);
@@ -112,10 +116,10 @@ impl RobotsTxt {
.to_ascii_lowercase()
.lines()
.find(|line| line.starts_with("sitemap:"))
- .map(|line| line.split(':').nth(1).unwrap().trim().to_string())
- .map(Url::from);
+ .map(|line| line.split(':').nth(1).unwrap().trim())
+ .and_then(|s| Url::parse(s).ok());
- Ok(Self { matcher, sitemap })
+ Self { matcher, sitemap }
}
}
@@ -130,8 +134,7 @@ mod tests {
r#"User-agent: StractSearch
Disallow: /test"#
.to_string(),
- )
- .unwrap();
+ );
assert!(!robots_txt
.matcher
diff --git a/core/src/crawler/warc_writer.rs b/core/src/crawler/warc_writer.rs
index 0c1fa821..e56740ef 100644
--- a/core/src/crawler/warc_writer.rs
+++ b/core/src/crawler/warc_writer.rs
@@ -102,7 +102,7 @@ async fn writer_task(mut rx: tokio::sync::mpsc::Receiver<WarcWriterMessage>, s3:
let warc_record = warc::WarcRecord {
request: warc::Request {
- url: datum.url.full(),
+ url: datum.url.to_string(),
},
response: warc::Response {
body: datum.body,
diff --git a/core/src/crawler/worker.rs b/core/src/crawler/worker.rs
index 8cd6b3e5..6fba83d5 100644
--- a/core/src/crawler/worker.rs
+++ b/core/src/crawler/worker.rs
@@ -24,12 +24,13 @@ use std::{
};
use tokio::sync::Mutex;
+use url::Url;
use crate::{
config::CrawlerConfig,
crawler::{JobResponse, Request, Response},
distributed::{retry_strategy::ExponentialBackoff, sonic},
- webpage::{Html, Url},
+ webpage::Html,
};
use super::{
@@ -60,7 +61,7 @@ impl Worker {
coordinator_host: SocketAddr,
) -> Result<Self> {
if config.politeness_factor < config.min_politeness_factor {
- return Err(Error::InvalidPolitenessFactor);
+ return Err(Error::InvalidPolitenessFactor.into());
}
let client = reqwest::Client::builder()
@@ -170,7 +171,9 @@ impl Worker {
continue;
}
- if retryable_url.url.is_empty() || retryable_url.url.site().is_empty() {
+ if retryable_url.url.host_str().is_none()
+ || !matches!(retryable_url.url.scheme(), "http" | "https")
+ {
continue;
}
@@ -182,7 +185,7 @@ impl Worker {
continue;
}
- let site = Site(retryable_url.url.site().to_string());
+ let site = Site(retryable_url.url.host_str().unwrap_or_default().to_string());
if job.fetch_sitemap && !crawled_sitemaps.contains(&site) {
crawled_sitemaps.insert(site.clone());
@@ -240,57 +243,69 @@ impl Worker {
if matches!(datum.status_code, 200 | 301 | 302) {
self.save_datum(datum.clone()).await;
- let html = Html::parse(&datum.body, &datum.url.full());
- let new_urls = html
- .all_links()
- .into_iter()
- .map(|link| Url::from(link.destination.full()).into_absolute(&datum.url))
- .filter(|url| {
- !url.path_ends_with(".pdf")
- && !url.path_ends_with(".jpg")
- && !url.path_ends_with(".zip")
- && !url.path_ends_with(".png")
- && !url.path_ends_with(".css")
- && !url.path_ends_with(".js")
- && !url.path_ends_with(".json")
- && !url.path_ends_with(".jsonp")
- && !url.path_ends_with(".woff2")
- && !url.path_ends_with(".woff")
- && !url.path_ends_with(".ttf")
- && !url.path_ends_with(".svg")
- && !url.path_ends_with(".gif")
- && !url.path_ends_with(".jpeg")
- && !url.path_ends_with(".ico")
- && !url.path_ends_with(".mp4")
- && !url.path_ends_with(".mp3")
- && !url.path_ends_with(".avi")
- && !url.path_ends_with(".mov")
- && !url.path_ends_with(".mpeg")
- && !url.path_ends_with(".webm")
- && !url.path_ends_with(".wav")
- && !url.path_ends_with(".flac")
- && !url.path_ends_with(".aac")
- && !url.path_ends_with(".ogg")
- && !url.path_ends_with(".m4a")
- && !url.path_ends_with(".m4v")
- })
- .collect();
-
- let url_res =
- if datum.status_code == 301 || datum.status_code == 302 || url != datum.url
- {
- UrlResponse::Redirected {
- url,
- new_url: datum.url,
+ match Html::parse(&datum.body, datum.url.as_str()) {
+ Ok(html) => {
+ let new_urls = html
+ .all_links()
+ .into_iter()
+ .map(|link| link.destination)
+ .filter(|url| {
+ !url.path().ends_with(".pdf")
+ && !url.path().ends_with(".jpg")
+ && !url.path().ends_with(".zip")
+ && !url.path().ends_with(".png")
+ && !url.path().ends_with(".css")
+ && !url.path().ends_with(".js")
+ && !url.path().ends_with(".json")
+ && !url.path().ends_with(".jsonp")
+ && !url.path().ends_with(".woff2")
+ && !url.path().ends_with(".woff")
+ && !url.path().ends_with(".ttf")
+ && !url.path().ends_with(".svg")
+ && !url.path().ends_with(".gif")
+ && !url.path().ends_with(".jpeg")
+ && !url.path().ends_with(".ico")
+ && !url.path().ends_with(".mp4")
+ && !url.path().ends_with(".mp3")
+ && !url.path().ends_with(".avi")
+ && !url.path().ends_with(".mov")
+ && !url.path().ends_with(".mpeg")
+ && !url.path().ends_with(".webm")
+ && !url.path().ends_with(".wav")
+ && !url.path().ends_with(".flac")
+ && !url.path().ends_with(".aac")
+ && !url.path().ends_with(".ogg")
+ && !url.path().ends_with(".m4a")
+ && !url.path().ends_with(".m4v")
+ })
+ .collect();
+
+ let url_res = if datum.status_code == 301
+ || datum.status_code == 302
+ || url != datum.url
+ {
+ UrlResponse::Redirected {
+ url,
+ new_url: datum.url,
+ }
+ } else {
+ UrlResponse::Success { url: datum.url }
+ };
+
+ ProcessedUrl {
+ new_urls,
+ response: url_res,
+ fetch_time: Duration::from_millis(datum.fetch_time_ms),
}
- } else {
- UrlResponse::Success { url: datum.url }
- };
-
- ProcessedUrl {
- new_urls,
- response: url_res,
- fetch_time: Duration::from_millis(datum.fetch_time_ms),
+ }
+ Err(_) => ProcessedUrl {
+ new_urls: Vec::new(),
+ response: UrlResponse::Failed {
+ url,
+ status_code: None,
+ },
+ fetch_time: Duration::from_millis(datum.fetch_time_ms),
+ },
}
} else {
if datum.status_code == 429 {
@@ -342,9 +357,9 @@ impl Worker {
.with_limit(Duration::from_secs(20))
.take(3);
- let mut res = Err(Error::FetchFailed(reqwest::StatusCode::IM_A_TEAPOT));
+ let mut res = Err(Error::FetchFailed(reqwest::StatusCode::IM_A_TEAPOT).into());
for time in backoff {
- if let Ok(cur_res) = self.client.get(url.full()).send().await {
+ if let Ok(cur_res) = self.client.get(url.to_string()).send().await {
res = Ok(cur_res);
break;
} else {
@@ -376,24 +391,24 @@ impl Worker {
// check if content length is too large
if let Some(content_length) = headers.get("content-length") {
if content_length.parse::<usize>().unwrap_or(0) > 10_000_000 {
- return Err(Error::ContentTooLarge);
+ return Err(Error::ContentTooLarge.into());
}
}
// check if content type is html
if let Some(content_type) = headers.get("content-type") {
if !content_type.contains("text/html") {
- return Err(Error::InvalidContentType(content_type.to_string()));
+ return Err(Error::InvalidContentType(content_type.to_string()).into());
}
}
let status_code = res.status().as_u16();
- let res_url = res.url().as_str().to_string();
+ let res_url = res.url().clone();
let body = res.text().await?;
Ok(CrawlDatum {
- url: Url::from(res_url).full().into(),
+ url: res_url,
status_code,
headers,
body,
@@ -464,23 +479,19 @@ impl Worker {
}
Ok(Event::Text(e)) => {
if in_sitemap && in_loc {
- let url = Url::from(e.unescape_and_decode(&reader).unwrap())
- .full()
- .into();
-
- tokio::time::sleep(Duration::from_millis(
- self.config.min_crawl_delay_ms,
- ))
- .await;
-
- urls.append(
- &mut self.urls_from_sitemap(url, depth + 1, max_depth).await,
- );
+ if let Ok(url) = Url::parse(&e.unescape_and_decode(&reader).unwrap()) {
+ urls.append(
+ &mut self.urls_from_sitemap(url, depth + 1, max_depth).await,
+ );
+ tokio::time::sleep(Duration::from_millis(
+ self.config.min_crawl_delay_ms,
+ ))
+ .await;
+ }
} else if in_url && in_loc {
- let url = Url::from(e.unescape_and_decode(&reader).unwrap())
- .full()
- .into();
- urls.push(url);
+ if let Ok(url) = Url::parse(&e.unescape_and_decode(&reader).unwrap()) {
+ urls.push(url);
+ }
}
}
Ok(Event::Eof) => break,
diff --git a/core/src/distributed/cluster.rs b/core/src/distributed/cluster.rs
index ee4ebee7..c5df507a 100644
--- a/core/src/distributed/cluster.rs
+++ b/core/src/distributed/cluster.rs
@@ -28,16 +28,7 @@ const CLUSTER_ID: &str = "stract-cluster";
const GOSSIP_INTERVAL: Duration = Duration::from_secs(1);
const SERVICE_KEY: &str = "service";
-#[derive(Debug, thiserror::Error)]
-pub enum Error {
- #[error("Anyhow")]
- Anyhow(#[from] anyhow::Error),
-
- #[error("serde json")]
- SerdeJson(#[from] serde_json::Error),
-}
-
-type Result<T> = std::result::Result<T, Error>;
+type Result<T> = std::result::Result<T, anyhow::Error>;
pub struct Cluster {
alive_nodes: Arc<RwLock<HashSet<Member>>>,
diff --git a/core/src/entity_index/entity.rs b/core/src/entity_index/entity.rs
index 7ec29058..527dc74f 100644
--- a/core/src/entity_index/entity.rs
+++ b/core/src/entity_index/entity.rs
@@ -19,8 +19,6 @@ use std::collections::{BTreeMap, HashSet};
use parse_wiki_text::Node;
use serde::{Deserialize, Serialize};
-use crate::webpage::Url;
-
#[derive(Debug)]
pub struct Paragraph {
pub title: Option<String>,
@@ -32,7 +30,7 @@ pub struct Entity {
pub title: String,
pub page_abstract: Span,
pub info: BTreeMap<String, Span>,
- pub image: Option<Url>,
+ pub image: Option<String>,
pub paragraphs: Vec<Paragraph>,
pub categories: HashSet<String>,
}
diff --git a/core/src/entity_index/mod.rs b/core/src/entity_index/mod.rs
index 9cc547e6..094178a2 100644
--- a/core/src/entity_index/mod.rs
+++ b/core/src/entity_index/mod.rs
@@ -31,13 +31,13 @@ use tantivy::{
DocAddress, IndexReader, IndexWriter, Searcher, Term,
};
use tracing::info;
+use url::Url;
use crate::{
image_downloader::{ImageDownloadJob, ImageDownloader},
image_store::{EntityImageStore, Image, ImageStore},
kv::{rocksdb_store::RocksDbStore, Kv},
tokenizer::Normal,
- webpage::Url,
Result,
};
@@ -119,8 +119,8 @@ fn entity_to_tantivy(entity: Entity, schema: &tantivy::schema::Schema) -> tantiv
doc
}
-fn wikipedify_url(url: &Url) -> Vec<Url> {
- let mut name = url.raw().replace(' ', "_");
+fn wikipedify_url(url: &str) -> Vec<Url> {
+ let mut name = url.replace(' ', "_");
if name.starts_with("File:") {
if let Some(index) = name.find("File:") {
@@ -130,18 +130,18 @@ fn wikipedify_url(url: &Url) -> Vec<Url> {
let hex = format!("{:?}", md5::compute(&name));
vec![
- format!(
+ Url::parse(&format!(
"https://upload.wikimedia.org/wikipedia/commons/{:}/{:}",
hex[0..1].to_string() + "/" + &hex[0..2],
name
- )
- .into(),
- format!(
+ ))
+ .unwrap(),
+ Url::parse(&format!(
"https://upload.wikimedia.org/wikipedia/en/{:}/{:}",
hex[0..1].to_string() + "/" + &hex[0..2],
name
- )
- .into(),
+ ))
+ .unwrap(),
]
}
@@ -213,10 +213,7 @@ impl EntityIndex {
}
if let Some(image) = entity.image.clone() {
- let image = wikipedify_url(&image)
- .into_iter()
- .filter(Url::is_valid_uri)
- .collect();
+ let image = wikipedify_url(&image).into_iter().collect();
self.image_downloader.schedule(ImageDownloadJob {
key: entity.title.clone(),
@@ -414,24 +411,22 @@ mod tests {
#[test]
fn wikipedia_image_url_aristotle() {
assert_eq!(
- wikipedify_url(&"Aristotle Altemps Inv8575.jpg".to_string().into())
+ wikipedify_url("Aristotle Altemps Inv8575.jpg")
.first()
.unwrap()
- .full(),
+ .as_str(),
"https://upload.wikimedia.org/wikipedia/commons/a/ae/Aristotle_Altemps_Inv8575.jpg"
- .to_string()
);
}
#[test]
fn wikipedia_image_url_with_file() {
assert_eq!(
- wikipedify_url(&"File:Aristotle Altemps Inv8575.jpg".to_string().into())
+ wikipedify_url("File:Aristotle Altemps Inv8575.jpg")
.first()
.unwrap()
- .full(),
+ .as_str(),
"https://upload.wikimedia.org/wikipedia/commons/a/ae/Aristotle_Altemps_Inv8575.jpg"
- .to_string()
);
}
diff --git a/core/src/entrypoint/alice.rs b/core/src/entrypoint/alice.rs
index 28cdd524..bb3f6dfd 100644
--- a/core/src/entrypoint/alice.rs
+++ b/core/src/entrypoint/alice.rs
@@ -167,7 +167,7 @@ pub async fn route(
)
}
-pub async fn run(config: AliceLocalConfig) -> Result<(), crate::alice::Error> {
+pub async fn run(config: AliceLocalConfig) -> Result<(), anyhow::Error> {
let addr: SocketAddr = config.host;
let key = base64::decode(config.encryption_key)?;
diff --git a/core/src/entrypoint/dmoz_parser.rs b/core/src/entrypoint/dmoz_parser.rs
index 41bd5ca6..c4b21ae6 100644
--- a/core/src/entrypoint/dmoz_parser.rs
+++ b/core/src/entrypoint/dmoz_parser.rs
@@ -16,10 +16,10 @@
use flate2::read::MultiGzDecoder;
use quick_xml::events::Event;
+use url::Url;
use crate::{
human_website_annotations::{Info, Topic},
- webpage::Url,
Result,
};
use std::{
@@ -127,9 +127,12 @@ pub fn parse<P: AsRef<Path>>(dmoz_file: P) -> Result<human_website_annotations::
let mut map = HashMap::new();
for page in PageIterator::from(reader) {
- let url = Url::from(page.url.clone());
+ let url = match Url::parse(&page.url) {
+ Ok(url) => url,
+ Err(_) => continue,
+ };
- if !url.is_homepage() {
+ if url.path() != "/" {
continue;
}
@@ -144,7 +147,7 @@ pub fn parse<P: AsRef<Path>>(dmoz_file: P) -> Result<human_website_annotations::
topic,
};
- map.insert(url.site().to_string(), info);
+ map.insert(url.host_str().unwrap().to_string(), info);
}
Ok(map.into())
diff --git a/core/src/entrypoint/entity.rs b/core/src/entrypoint/entity.rs
index 709fb574..f25cbfa4 100644
--- a/core/src/entrypoint/entity.rs
+++ b/core/src/entrypoint/entity.rs
@@ -323,7 +323,7 @@ impl EntityBuilder {
}
let mut info = self.extract_info();
- let image = info.remove("image").map(|span| span.text.into());
+ let image = info.remove("image").map(|span| span.text);
let (page_abstract, paragraphs) = self.extract_text();
@@ -387,7 +387,7 @@ mod tests {
assert_eq!(
entity.image,
- Some("Aristotle Altemps Inv8575.jpg".to_string().into())
+ Some("Aristotle Altemps Inv8575.jpg".to_string())
);
assert!(
@@ -461,10 +461,7 @@ mod tests {
);
assert_eq!(entity.info.get("death_date"), None);
- assert_eq!(
- entity.image,
- Some("President Barack Obama.jpg".to_string().into())
- );
+ assert_eq!(entity.image, Some("President Barack Obama.jpg".to_string()));
assert!(
entity.page_abstract.text.trim().starts_with("Barack Hussein Obama II ( ; born August 4, 1961) is an American politician who served as the 44th president of the United States from 2009 to 2017. A member of the Democratic Party, he was the first African-American president of the United States."));
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 5f083365..659436ae 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -16,6 +16,7 @@
use chrono::Utc;
use std::path::Path;
use std::thread;
+use url::Url;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
@@ -126,7 +127,14 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
None => true,
})
{
- let mut html = Html::parse_without_text(&record.response.body, &record.request.url);
+ let mut html =
+ match Html::parse_without_text(&record.response.body, &record.request.url) {
+ Ok(html) => html,
+ Err(err) => {
+ debug!("error parsing html: {:?}", err);
+ continue;
+ }
+ };
if html.is_no_index() {
continue;
@@ -169,8 +177,14 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
.ingoing_edges(Node::from(html.url()))
.into_iter()
.map(|edge| Link {
- source: edge.from.name.into(),
- destination: edge.to.name.into(),
+ source: Url::parse(
+ &("http://".to_string() + edge.from.name.as_str()),
+ )
+ .unwrap(),
+ destination: Url::parse(
+ &("http://".to_string() + edge.to.name.as_str()),
+ )
+ .unwrap(),
text: edge.label,
})
.collect()
@@ -214,7 +228,9 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
let mut dmoz_description = None;
if let Some(mapper) = worker.topics.as_ref() {
- if let Some(info) = mapper.get(&html.url().site().to_string()) {
+ if let Some(info) =
+ mapper.get(&html.url().host_str().unwrap_or_default().to_string())
+ {
dmoz_description = Some(info.description.clone())
}
}
diff --git a/core/src/entrypoint/search_server.rs b/core/src/entrypoint/search_server.rs
index d5139d90..0016d60e 100644
--- a/core/src/entrypoint/search_server.rs
+++ b/core/src/entrypoint/search_server.rs
@@ -33,7 +33,6 @@ use crate::{
models::{lambdamart::LambdaMART, linear::LinearRegression},
},
searcher::{self, LocalSearcher},
- webpage::Url,
Result,
};
@@ -125,7 +124,7 @@ pub async fn run(config: config::SearchServerConfig) -> Result<()> {
for url in urls {
if let Some(homepage) = local_searcher.get_homepage(url) {
if let Some(desc) = homepage.description() {
- result.insert(Url::from(url.clone()), desc.clone());
+ result.insert(url.clone(), desc.clone());
}
}
}
diff --git a/core/src/entrypoint/webgraph.rs b/core/src/entrypoint/webgraph.rs
index a4928652..96a2de9e 100644
--- a/core/src/entrypoint/webgraph.rs
+++ b/core/src/entrypoint/webgraph.rs
@@ -101,14 +101,25 @@ impl WebgraphWorker {
if let Ok(file) = WarcFile::open(&path) {
for record in file.records().flatten() {
- let webpage =
- Html::parse_without_text(&record.response.body, &record.request.url);
+ let webpage = match Html::parse_without_text(
+ &record.response.body,
+ &record.request.url,
+ ) {
+ Ok(webpage) => webpage,
+ Err(err) => {
+ tracing::error!("error parsing webpage: {}", err);
+ continue;
+ }
+ };
+
for link in webpage
.anchor_links()
.into_iter()
- .filter(|link| matches!(link.destination.protocol(), "http" | "https"))
+ .filter(|link| matches!(link.destination.scheme(), "http" | "https"))
.filter(|link| link.source.domain() != link.destination.domain())
- .filter(|link| link.matches_url_regex())
+ .filter(|link| {
+ link.source.domain().is_some() && link.destination.domain().is_some()
+ })
{
let source = link.source.clone();
let mut destination = link.destination.clone();
diff --git a/core/src/entrypoint/webgraph_server.rs b/core/src/entrypoint/webgraph_server.rs
index aecc196a..c24e4f84 100644
--- a/core/src/entrypoint/webgraph_server.rs
+++ b/core/src/entrypoint/webgraph_server.rs
@@ -21,6 +21,7 @@ use itertools::Itertools;
use serde::Deserialize;
use serde::Serialize;
use tracing::info;
+use url::Url;
use crate::config;
use crate::distributed::cluster::Cluster;
@@ -32,7 +33,6 @@ use crate::searcher::DistributedSearcher;
use crate::similar_sites::SimilarSitesFinder;
use crate::webgraph::Node;
use crate::webgraph::WebgraphBuilder;
-use crate::webpage::Url;
use crate::Result;
#[derive(Serialize, Deserialize)]
@@ -89,7 +89,9 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
let urls = similar_sites
.iter()
- .map(|s| s.node.name.clone())
+ .map(|s| {
+ Url::parse(&("http://".to_string() + s.node.name.as_str())).unwrap()
+ })
.collect_vec();
let descriptions = searcher.get_homepage_descriptions(&urls).await;
@@ -98,7 +100,10 @@ pub async fn run(config: config::WebgraphServerConfig) -> Result<()> {
.into_iter()
.map(|site| {
let description = descriptions
- .get(&Url::from(site.node.name.clone()))
+ .get(
+ &Url::parse(&("http://".to_string() + site.node.name.as_str()))
+ .unwrap(),
+ )
.cloned();
ScoredSite {
diff --git a/core/src/executor.rs b/core/src/executor.rs
index 4244bd95..597ae069 100644
--- a/core/src/executor.rs
+++ b/core/src/executor.rs
@@ -90,7 +90,8 @@ impl Executor {
if results.len() != num_jobs {
return Err(Error::InternalError(
"At least one of the scheduled jobs failed.".to_string(),
- ));
+ )
+ .into());
}
Ok(results)
diff --git a/core/src/image_downloader.rs b/core/src/image_downloader.rs
index 3a941d60..b187ecef 100644
--- a/core/src/image_downloader.rs
+++ b/core/src/image_downloader.rs
@@ -17,11 +17,11 @@ use futures::StreamExt;
use std::{collections::HashSet, hash::Hash, time::Duration};
use serde::Serialize;
+use url::Url;
use crate::{
distributed::retry_strategy::ExponentialBackoff,
image_store::{Image, ImageStore},
- webpage::Url,
};
#[derive(Clone, Debug)]
@@ -40,19 +40,15 @@ where
{
async fn download(self) -> Option<DownloadedImage<K>> {
for url in &self.urls {
- if !url.is_valid_uri() {
- continue;
- }
-
for duration in ExponentialBackoff::from_millis(10).take(4) {
- if let Some(image) = url
- .download_bytes(self.timeout.unwrap_or_else(|| Duration::from_secs(20)))
- .await
- .and_then(|bytes| Image::from_bytes(bytes).ok())
- .map(|image| DownloadedImage {
- image,
- key: self.key.clone(),
- })
+ if let Some(image) =
+ download_bytes(url, self.timeout.unwrap_or_else(|| Duration::from_secs(20)))
+ .await
+ .and_then(|bytes| Image::from_bytes(bytes).ok())
+ .map(|image| DownloadedImage {
+ image,
+ key: self.key.clone(),
+ })
{
return Some(image);
}
@@ -65,6 +61,20 @@ where
}
}
+async fn download_bytes(url: &Url, timeout: Duration) -> Option<Vec<u8>> {
+ let client = reqwest::Client::builder().timeout(timeout).build().unwrap();
+
+ tracing::debug!("downloading {}", url);
+
+ match client.get(url.as_str()).send().await {
+ Ok(res) => {
+ let bytes = res.bytes().await.ok()?.to_vec();
+ Some(bytes)
+ }
+ Err(_) => None,
+ }
+}
+
impl<K> Hash for ImageDownloadJob<K>
where
K: std::fmt::Debug + Serialize + Hash + PartialEq + Eq + Clone,
diff --git a/core/src/improvement.rs b/core/src/improvement.rs
index b8f479cc..2b8ef358 100644
--- a/core/src/improvement.rs
+++ b/core/src/improvement.rs
@@ -20,9 +20,10 @@ use chrono::{DateTime, NaiveDate, Timelike, Utc};
use scylla::{prepared_statement::PreparedStatement, SessionBuilder};
use thiserror::Error;
use tokio::{sync::Mutex, time};
+use url::Url;
use uuid::Uuid;
-use crate::{api::improvement::AliceConversation, leaky_queue::LeakyQueue, webpage::Url};
+use crate::{api::improvement::AliceConversation, leaky_queue::LeakyQueue};
#[derive(Debug, Error)]
enum Error {
diff --git a/core/src/index.rs b/core/src/index.rs
index 868397ee..1f9e8d3b 100644
--- a/core/src/index.rs
+++ b/core/src/index.rs
@@ -23,6 +23,7 @@ use serde::{Deserialize, Serialize};
use tantivy::collector::Collector;
use tantivy::schema::Schema;
use tantivy::tokenizer::TokenizerManager;
+use url::Url;
use crate::directory::{self, DirEntry};
use crate::inverted_index::{self, InvertedIndex};
@@ -142,7 +143,7 @@ impl Index {
self.inverted_index.get_webpage(url)
}
- pub fn get_homepage(&self, url: &str) -> Option<inverted_index::RetrievedWebpage> {
+ pub fn get_homepage(&self, url: &Url) -> Option<inverted_index::RetrievedWebpage> {
self.inverted_index.get_homepage(url)
}
}
@@ -197,9 +198,10 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -209,9 +211,11 @@ mod tests {
</body>
</html>
"#
- ),
- "https://www.example.com",
- ))
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().unwrap();
@@ -235,7 +239,7 @@ mod tests {
assert_eq!(result.num_hits, 1);
assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.example.com");
+ assert_eq!(result.webpages[0].url, "https://www.example.com/");
}
#[test]
@@ -243,9 +247,10 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -255,15 +260,18 @@ mod tests {
</body>
</html>
"#,
- crate::rand_words(100)
- ),
- "https://www.first.com",
- ))
+ crate::rand_words(100)
+ ),
+ "https://www.first.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test test website</title>
@@ -273,15 +281,18 @@ mod tests {
</body>
</html>
"#,
- crate::rand_words(100)
- ),
- "https://www.second.com",
- ))
+ crate::rand_words(100)
+ ),
+ "https://www.second.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test test test website</title>
@@ -291,10 +302,12 @@ mod tests {
</body>
</html>
"#,
- crate::rand_words(100)
- ),
- "https://www.third.com",
- ))
+ crate::rand_words(100)
+ ),
+ "https://www.third.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().unwrap();
diff --git a/core/src/inverted_index.rs b/core/src/inverted_index.rs
index 5ae6d04c..00d36a5e 100644
--- a/core/src/inverted_index.rs
+++ b/core/src/inverted_index.rs
@@ -33,6 +33,7 @@ use tantivy::merge_policy::NoMergePolicy;
use tantivy::schema::Schema;
use tantivy::tokenizer::TokenizerManager;
use tantivy::{Document, IndexReader, IndexWriter, SegmentMeta};
+use url::Url;
use crate::collector::Hashes;
use crate::config::SnippetConfig;
@@ -46,7 +47,7 @@ use crate::search_ctx::Ctx;
use crate::tokenizer::{BigramTokenizer, Identity, TrigramTokenizer};
use crate::webgraph::NodeID;
use crate::webpage::region::Region;
-use crate::webpage::{schema_org, Url, Webpage};
+use crate::webpage::{schema_org, Webpage};
use crate::Result;
use crate::{combine_u64s, snippet};
use crate::{schema::create_schema, tokenizer::Tokenizer};
@@ -455,13 +456,14 @@ impl InvertedIndex {
}
pub(crate) fn get_webpage(&self, url: &str) -> Option<RetrievedWebpage> {
+ let url = Url::parse(url).ok()?;
let tv_searcher = self.reader.searcher();
let field = tv_searcher
.schema()
.get_field(Field::Text(TextField::UrlNoTokenizer).name())
.unwrap();
- let term = tantivy::Term::from_field_text(field, url);
+ let term = tantivy::Term::from_field_text(field, url.as_str());
let query = tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic);
@@ -473,15 +475,14 @@ impl InvertedIndex {
.map(|(_, doc)| self.retrieve_doc(doc.into(), &tv_searcher).unwrap())
}
- pub(crate) fn get_homepage(&self, url: &str) -> Option<RetrievedWebpage> {
- let url = Url::from(url.to_string());
+ pub(crate) fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {
let tv_searcher = self.reader.searcher();
let field = tv_searcher
.schema()
.get_field(Field::Text(TextField::SiteIfHomepageNoTokenizer).name())
.unwrap();
- let term = tantivy::Term::from_field_text(field, url.site());
+ let term = tantivy::Term::from_field_text(field, url.host_str().unwrap_or_default());
let query = tantivy::query::TermQuery::new(term, tantivy::schema::IndexRecordOption::Basic);
@@ -665,9 +666,10 @@ mod tests {
assert_eq!(result.num_docs, 0);
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -677,9 +679,11 @@ mod tests {
</body>
</html>
"#
- ),
- "https://www.example.com",
- ))
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let ctx = index.local_search_ctx();
@@ -694,7 +698,7 @@ mod tests {
search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.example.com");
+ assert_eq!(result.documents[0].url, "https://www.example.com/");
}
#[test]
@@ -702,9 +706,10 @@ mod tests {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -714,9 +719,11 @@ mod tests {
</body>
</html>
"#
- ),
- "https://www.example.com",
- ))
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -748,9 +755,10 @@ mod tests {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Website for runners</title>
@@ -760,9 +768,11 @@ mod tests {
</body>
</html>
"#
- ),
- "https://www.example.com",
- ))
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -785,7 +795,7 @@ mod tests {
let result =
search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.example.com");
+ assert_eq!(result.documents[0].url, "https://www.example.com/");
}
#[test]
@@ -793,9 +803,10 @@ mod tests {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Fast runner</title>
@@ -805,9 +816,11 @@ mod tests {
</body>
</html>
"#
- ),
- "https://www.example.com",
- ))
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -830,7 +843,7 @@ mod tests {
let result =
search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.example.com");
+ assert_eq!(result.documents[0].url, "https://www.example.com/");
}
#[test]
@@ -838,9 +851,10 @@ mod tests {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Website A</title>
@@ -849,9 +863,11 @@ mod tests {
{CONTENT}
</html>
"#
- ),
- "https://www.a.com",
- ))
+ ),
+ "https://www.a.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index
.insert(Webpage {
@@ -869,10 +885,11 @@ mod tests {
"#
),
"https://www.b.com",
- ),
+ )
+ .unwrap(),
backlinks: vec![Link {
- source: "https://www.a.com".to_string().into(),
- destination: "https://www.b.com".to_string().into(),
+ source: Url::parse("https://www.a.com/").unwrap(),
+ destination: Url::parse("https://www.b.com/").unwrap(),
text: "B site is great".to_string(),
}],
host_centrality: 1.0,
@@ -910,7 +927,7 @@ mod tests {
.sort_by(|a, b| a.url.partial_cmp(&b.url).unwrap());
assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.a.com");
+ assert_eq!(result.documents[0].url, "https://www.a.com/");
}
#[test]
@@ -921,9 +938,10 @@ mod tests {
let dedup_s = crate::rand_words(100);
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Website for runners</title>
@@ -933,9 +951,11 @@ mod tests {
</body>
</html>
"#
- ),
- "https://www.example.com",
- ))
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
}
@@ -967,9 +987,10 @@ mod tests {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>News website</title>
@@ -979,9 +1000,11 @@ mod tests {
</body>
</html>
"#
- ),
- "https://www.dr.dk",
- ))
+ ),
+ "https://www.dr.dk",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1004,7 +1027,7 @@ mod tests {
let result =
search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.dr.dk");
+ assert_eq!(result.documents[0].url, "https://www.dr.dk/");
}
#[test]
@@ -1012,9 +1035,10 @@ mod tests {
let mut index1 = InvertedIndex::temporary().expect("Unable to open index");
index1
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -1024,18 +1048,21 @@ mod tests {
</body>
</html>
"#,
- crate::rand_words(100)
- ),
- "https://www.example.com",
- ))
+ crate::rand_words(100)
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
let mut index2 = InvertedIndex::temporary().expect("Unable to open index");
index2
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -1045,10 +1072,12 @@ mod tests {
</body>
</html>
"#,
- crate::rand_words(100)
- ),
- "https://www.example.com",
- ))
+ crate::rand_words(100)
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
let mut index = index1.merge(index2);
@@ -1074,8 +1103,8 @@ mod tests {
search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.num_docs, 2);
assert_eq!(result.documents.len(), 2);
- assert_eq!(result.documents[0].url, "https://www.example.com");
- assert_eq!(result.documents[1].url, "https://www.example.com");
+ assert_eq!(result.documents[0].url, "https://www.example.com/");
+ assert_eq!(result.documents[1].url, "https://www.example.com/");
}
#[test]
@@ -1104,9 +1133,10 @@ mod tests {
assert_eq!(result.num_docs, 0);
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -1116,9 +1146,11 @@ mod tests {
</body>
</html>
"#
- ),
- "https://www.example.com",
- ))
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1132,7 +1164,7 @@ mod tests {
search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.example.com");
+ assert_eq!(result.documents[0].url, "https://www.example.com/");
}
#[test]
@@ -1140,9 +1172,10 @@ mod tests {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -1152,9 +1185,11 @@ mod tests {
</body>
</html>
"#
- ),
- "https://www.example.com#tag",
- ))
+ ),
+ "https://www.example.com#tag",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1178,7 +1213,7 @@ mod tests {
search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.num_docs, 1);
assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.example.com");
+ assert_eq!(result.documents[0].url, "https://www.example.com/");
}
#[test]
@@ -1186,9 +1221,10 @@ mod tests {
let mut index = InvertedIndex::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>News website</title>
@@ -1198,15 +1234,18 @@ mod tests {
</body>
</html>
"#
- ),
- "https://www.dr.xyz",
- ))
+ ),
+ "https://www.dr.xyz",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>News website</title>
@@ -1216,9 +1255,11 @@ mod tests {
</body>
</html>
"#
- ),
- "https://www.dr.dk",
- ))
+ ),
+ "https://www.dr.dk",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1242,7 +1283,7 @@ mod tests {
let result =
search(&index, &query, &ctx, ranker.collector(ctx.clone())).expect("Search failed");
assert_eq!(result.documents.len(), 1);
- assert_eq!(result.documents[0].url, "https://www.dr.dk");
+ assert_eq!(result.documents[0].url, "https://www.dr.dk/");
}
#[test]
@@ -1269,7 +1310,7 @@ mod tests {
"#
),
"https://www.example.com",
- ))
+ ).unwrap())
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -1348,14 +1389,14 @@ mod tests {
"#
),
"https://www.example.com",
- ))
+ ).unwrap())
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let webpage = index.get_webpage("https://www.example.com").unwrap();
assert_eq!(webpage.title, "News website".to_string());
- assert_eq!(webpage.url, "https://www.example.com".to_string());
+ assert_eq!(webpage.url, "https://www.example.com/".to_string());
}
#[test]
@@ -1383,13 +1424,15 @@ mod tests {
"#
),
"https://www.example.com",
- ))
+ ).unwrap())
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
- let webpage = index.get_homepage("https://www.example.com").unwrap();
+ let webpage = index
+ .get_homepage(&Url::parse("https://www.example.com").unwrap())
+ .unwrap();
assert_eq!(webpage.title, "News website".to_string());
- assert_eq!(webpage.url, "https://www.example.com".to_string());
+ assert_eq!(webpage.url, "https://www.example.com/".to_string());
}
}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index 5ddbc29e..0074590e 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -21,10 +21,7 @@
#![allow(clippy::cast_precision_loss)]
#![allow(clippy::missing_errors_doc)]
-use std::io;
-use std::num::ParseIntError;
use std::path::PathBuf;
-use tantivy::TantivyError;
use thiserror::Error;
pub mod entrypoint;
@@ -83,113 +80,38 @@ mod widgets;
#[derive(Error, Debug)]
pub enum Error {
- #[error("Failed to download object from HTTP")]
- HTTPDownloadError(#[from] reqwest::Error),
-
- #[error("Got an IO error")]
- IOError(#[from] io::Error),
-
- #[error("Not valid UTF8")]
- FromUTF8(#[from] std::string::FromUtf8Error),
-
#[error("Failed to parse WARC file")]
WarcParse(&'static str),
- #[error("Could not parse string to int")]
- IntParse(#[from] ParseIntError),
-
- #[error("Encountered a tantivy error")]
- Tantivy(#[from] TantivyError),
-
#[error("Encountered an empty required field when converting to tantivy")]
EmptyField(&'static str),
#[error("Parsing error")]
ParsingError(String),
- #[error("Error while serializing/deserializing to/from bytes")]
- Serialization(#[from] bincode::Error),
-
- #[error("Error executing distributed jobs")]
- MapReduce(#[from] mapreduce::Error),
-
#[error("Failed to download warc files after all retries")]
DownloadFailed,
- #[error("Encountered an error when reading CSV file")]
- Csv(#[from] csv::Error),
-
- #[error("Encountered an error in the FST crate")]
- Fst(#[from] fst::Error),
-
- #[error("Image error")]
- Image(#[from] image::ImageError),
-
- #[error("XML parser error")]
- XML(#[from] quick_xml::Error),
-
- #[error("Spell dictionary error")]
- Spell(#[from] crate::spell::dictionary::DictionaryError),
-
- #[error("Optic")]
- Optics(#[from] optics::Error),
-
#[error("Query cannot be completely empty")]
EmptyQuery,
#[error("Unknown region")]
UnknownRegion,
- #[error("String is not float")]
- ParseFloat(#[from] std::num::ParseFloatError),
-
- #[error("Could not open inverted-index directory")]
- Directory(#[from] tantivy::directory::error::OpenDirectoryError),
-
- #[error("Could not convert to/from JSON")]
- Json(#[from] serde_json::Error),
-
#[error("Unknown CLI option")]
UnknownCLIOption,
#[error("The stackoverflow schema was not structured as expected")]
InvalidStackoverflowSchema,
- #[error("Rayong thread pool builder")]
- RayonThreadPool(#[from] rayon::ThreadPoolBuildError),
-
#[error("Internal error")]
InternalError(String),
- #[error("Distributed searcher")]
- DistributedSearcher(#[from] searcher::distributed::Error),
-
- #[error("Cluster")]
- Cluster(#[from] crate::distributed::cluster::Error),
-
- #[error("Lambdamart")]
- LambdaMART(#[from] ranking::models::lambdamart::Error),
-
- #[error("Summarizer")]
- Summarizer(#[from] crate::summarizer::Error),
-
- #[error("Alice")]
- Alice(#[from] crate::alice::Error),
-
- #[error("Fact check model")]
- FactCheckModel(#[from] crate::fact_check_model::Error),
-
- #[error("Crawler")]
- Crawler(#[from] crate::crawler::Error),
-
- #[error("S3")]
- S3(#[from] s3::error::S3Error),
-
#[error("Unknown webpage robots meta tag")]
UnknownRobotsMetaTag,
}
-pub(crate) type Result<T> = std::result::Result<T, Error>;
+pub(crate) type Result<T> = std::result::Result<T, anyhow::Error>;
// taken from https://docs.rs/sled/0.34.7/src/sled/config.rs.html#445
pub fn gen_temp_path() -> PathBuf {
diff --git a/core/src/query/mod.rs b/core/src/query/mod.rs
index 74e2bfa9..b779e0b8 100644
--- a/core/src/query/mod.rs
+++ b/core/src/query/mod.rs
@@ -338,8 +338,9 @@ mod tests {
};
index
- .insert(Webpage::new(
- r#"
+ .insert(
+ Webpage::new(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -349,12 +350,15 @@ mod tests {
</body>
</html>
"#,
- "https://www.first.com",
- ))
+ "https://www.first.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index
- .insert(Webpage::new(
- r#"
+ .insert(
+ Webpage::new(
+ r#"
<html>
<head>
<title>Test test</title>
@@ -364,8 +368,10 @@ mod tests {
</body>
</html>
"#,
- "https://www.second.com",
- ))
+ "https://www.second.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
@@ -373,7 +379,7 @@ mod tests {
let result = searcher.search(&query).expect("Search failed");
assert_eq!(result.num_hits, 1);
assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.second.com");
+ assert_eq!(result.webpages[0].url, "https://www.second.com/");
}
#[test]
@@ -381,8 +387,9 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- r#"
+ .insert(
+ Webpage::new(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -392,12 +399,15 @@ mod tests {
</body>
</html>
"#,
- "https://www.first.com",
- ))
+ "https://www.first.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index
- .insert(Webpage::new(
- r#"
+ .insert(
+ Webpage::new(
+ r#"
<html>
<head>
<title>Test test</title>
@@ -407,8 +417,10 @@ mod tests {
</body>
</html>
"#,
- "https://www.second.com",
- ))
+ "https://www.second.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
@@ -420,7 +432,7 @@ mod tests {
let result = searcher.search(&query).expect("Search failed");
assert_eq!(result.num_hits, 1);
assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.first.com");
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
let query = SearchQuery {
query: "test site:www.first.com".to_string(),
@@ -429,7 +441,7 @@ mod tests {
let result = searcher.search(&query).expect("Search failed");
assert_eq!(result.num_hits, 1);
assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.first.com");
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
let query = SearchQuery {
query: "test -site:first.com".to_string(),
@@ -438,7 +450,7 @@ mod tests {
let result = searcher.search(&query).expect("Search failed");
assert_eq!(result.num_hits, 1);
assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.second.com");
+ assert_eq!(result.webpages[0].url, "https://www.second.com/");
}
#[test]
@@ -446,8 +458,9 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- r#"
+ .insert(
+ Webpage::new(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -457,12 +470,15 @@ mod tests {
</body>
</html>
"#,
- "https://www.first.com",
- ))
+ "https://www.first.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index
- .insert(Webpage::new(
- r#"
+ .insert(
+ Webpage::new(
+ r#"
<html>
<head>
<title>Test test</title>
@@ -472,8 +488,10 @@ mod tests {
</body>
</html>
"#,
- "https://www.second.com",
- ))
+ "https://www.second.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
@@ -485,7 +503,7 @@ mod tests {
let result = searcher.search(&query).expect("Search failed");
assert_eq!(result.num_hits, 1);
assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.first.com");
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
}
#[test]
@@ -493,8 +511,9 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- r#"
+ .insert(
+ Webpage::new(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -504,12 +523,15 @@ mod tests {
</body>
</html>
"#,
- "https://www.first.com/forum",
- ))
+ "https://www.first.com/forum",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index
- .insert(Webpage::new(
- r#"
+ .insert(
+ Webpage::new(
+ r#"
<html>
<head>
<title>Test test</title>
@@ -519,8 +541,10 @@ mod tests {
</body>
</html>
"#,
- "https://www.second.com",
- ))
+ "https://www.second.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
@@ -576,9 +600,10 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -588,15 +613,18 @@ mod tests {
</body>
</html>
"#,
- rand_words(1000)
- ),
- "https://www.the-first.com",
- ))
+ rand_words(1000)
+ ),
+ "https://www.the-first.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test test</title>
@@ -606,10 +634,12 @@ mod tests {
</body>
</html>
"#,
- rand_words(1000)
- ),
- "https://www.second.com",
- ))
+ rand_words(1000)
+ ),
+ "https://www.second.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
@@ -629,7 +659,7 @@ mod tests {
let result = searcher.search(&query).expect("Search failed");
assert_eq!(result.num_hits, 1);
assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.the-first.com");
+ assert_eq!(result.webpages[0].url, "https://www.the-first.com/");
let query = SearchQuery {
query: "test site:www.the-first.com".to_string(),
@@ -638,7 +668,7 @@ mod tests {
let result = searcher.search(&query).expect("Search failed");
assert_eq!(result.num_hits, 1);
assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.the-first.com");
+ assert_eq!(result.webpages[0].url, "https://www.the-first.com/");
}
#[test]
@@ -646,9 +676,10 @@ mod tests {
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test website</title>
@@ -658,15 +689,18 @@ mod tests {
</body>
</html>
"#,
- rand_words(1000)
- ),
- "https://www.first.com",
- ))
+ rand_words(1000)
+ ),
+ "https://www.first.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Test test</title>
@@ -676,10 +710,12 @@ mod tests {
</body>
</html>
"#,
- rand_words(1000)
- ),
- "https://www.second.com",
- ))
+ rand_words(1000)
+ ),
+ "https://www.second.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
let searcher = LocalSearcher::from(index);
@@ -691,7 +727,7 @@ mod tests {
let result = searcher.search(&query).expect("Search failed");
assert_eq!(result.num_hits, 1);
assert_eq!(result.webpages.len(), 1);
- assert_eq!(result.webpages[0].url, "https://www.first.com");
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
let query = SearchQuery {
query: "\"Test website\" site:www.second.com".to_string(),
diff --git a/core/src/query/optic.rs b/core/src/query/optic.rs
index 8871fc38..e3a3ec8c 100644
--- a/core/src/query/optic.rs
+++ b/core/src/query/optic.rs
@@ -144,7 +144,7 @@ impl AsTantivyQuery for Matching {
MatchLocation::Site => ConstQuery::new(
PatternQuery::new(
self.pattern.clone(),
- TextField::Site,
+ TextField::SiteWithout,
schema,
fastfield_reader.clone(),
)
@@ -259,7 +259,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.a.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -287,7 +287,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.b.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.01,
page_centrality: 0.0,
@@ -311,8 +311,8 @@ mod tests {
.webpages;
assert_eq!(res.len(), 2);
- assert_eq!(res[0].url, "https://www.b.com");
- assert_eq!(res[1].url, "https://www.a.com");
+ assert_eq!(res[0].url, "https://www.b.com/");
+ assert_eq!(res[1].url, "https://www.a.com/");
let res = searcher
.search(&SearchQuery {
@@ -336,7 +336,7 @@ mod tests {
.webpages;
assert_eq!(res.len(), 1);
- assert_eq!(res[0].url, "https://www.a.com");
+ assert_eq!(res[0].url, "https://www.a.com/");
let res = searcher
.search(&SearchQuery {
@@ -360,8 +360,8 @@ mod tests {
.webpages;
assert_eq!(res.len(), 2);
- assert_eq!(res[0].url, "https://www.a.com");
- assert_eq!(res[1].url, "https://www.b.com");
+ assert_eq!(res[0].url, "https://www.a.com/");
+ assert_eq!(res[1].url, "https://www.b.com/");
}
#[test]
@@ -385,7 +385,7 @@ mod tests {
"#
),
"https://www.a.com/this/is/a/pattern",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -412,7 +412,7 @@ mod tests {
"#
),
"https://www.b.com/this/is/b/pattern",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
@@ -488,7 +488,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.a.com/this/is/a/pattern",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -516,7 +516,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.b.com/this/is/b/pattern",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
@@ -544,7 +544,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.c.com/this/is/c/pattern",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
@@ -647,7 +647,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.a.com/this/is/a/pattern",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -675,7 +675,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.b.com/this/is/b/pattern",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
@@ -703,7 +703,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.c.com/this/is/c/pattern",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0002,
page_centrality: 0.0,
@@ -778,7 +778,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.a.com/",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -821,7 +821,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.b.com/",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0001,
page_centrality: 0.0,
@@ -858,7 +858,7 @@ mod tests {
.webpages;
assert_eq!(res.len(), 1);
- assert_eq!(res[0].url, "https://www.b.com");
+ assert_eq!(res[0].url, "https://www.b.com/");
let res = searcher
.search(&SearchQuery {
@@ -882,7 +882,7 @@ mod tests {
.webpages;
assert_eq!(res.len(), 1);
- assert_eq!(res[0].url, "https://www.b.com");
+ assert_eq!(res[0].url, "https://www.b.com/");
let res = searcher
.search(&SearchQuery {
@@ -906,7 +906,7 @@ mod tests {
.webpages;
assert_eq!(res.len(), 1);
- assert_eq!(res[0].url, "https://www.a.com");
+ assert_eq!(res[0].url, "https://www.a.com/");
let res = searcher
.search(&SearchQuery {
@@ -930,7 +930,7 @@ mod tests {
.webpages;
assert_eq!(res.len(), 1);
- assert_eq!(res[0].url, "https://www.b.com");
+ assert_eq!(res[0].url, "https://www.b.com/");
}
#[test]
@@ -955,7 +955,7 @@ mod tests {
crate::rand_words(100)
),
"https://chat.stackoverflow.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -1029,7 +1029,7 @@ mod tests {
crate::rand_words(100)
),
"https://a.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -1057,8 +1057,8 @@ mod tests {
"#,
crate::rand_words(100)
),
- "https://b.com",
- ),
+ "https://b.com/",
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -1100,7 +1100,7 @@ mod tests {
.webpages;
assert_eq!(res.len(), 1);
- assert_eq!(res[0].url, "https://b.com");
+ assert_eq!(res[0].url, "https://b.com/");
}
#[test]
@@ -1112,7 +1112,7 @@ mod tests {
html: Html::parse(
include_str!("../../testcases/schema_org/infinity_war.html"),
"https://a.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -1135,7 +1135,7 @@ mod tests {
.webpages;
assert!(!res.is_empty());
- assert_eq!(&res[0].url, "https://a.com");
+ assert_eq!(&res[0].url, "https://a.com/");
let res = searcher
.search(&SearchQuery {
@@ -1170,7 +1170,7 @@ mod tests {
crate::rand_words(1000)
),
"https://example.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -1192,7 +1192,7 @@ mod tests {
.unwrap()
.webpages;
assert_eq!(res.len(), 1);
- assert_eq!(res[0].url, "https://example.com");
+ assert_eq!(res[0].url, "https://example.com/");
let res = searcher
.search(&SearchQuery {
@@ -1331,7 +1331,7 @@ mod tests {
crate::rand_words(1000)
),
"https://example.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -1404,7 +1404,7 @@ mod tests {
crate::rand_words(1000)
),
"https://example.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -1464,7 +1464,7 @@ mod tests {
crate::rand_words(1000)
),
"https://example.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -1493,7 +1493,7 @@ mod tests {
crate::rand_words(1000)
),
"https://example.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
@@ -1590,8 +1590,8 @@ mod tests {
</body>
</html>
"#,
- "https://example.com",
- ),
+ "https://example.com/",
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
page_centrality: 0.0,
diff --git a/core/src/query/pattern_query.rs b/core/src/query/pattern_query.rs
index 029ef073..95c50fe4 100644
--- a/core/src/query/pattern_query.rs
+++ b/core/src/query/pattern_query.rs
@@ -182,7 +182,7 @@ fn can_optimize_site_domain(patterns: &[PatternPart], field: Field) -> bool {
&& patterns[1..patterns.len() - 1]
.iter()
.all(|pattern| matches!(pattern, PatternPart::Raw(_)))
- && (matches!(field, Field::Text(TextField::Site))
+ && (matches!(field, Field::Text(TextField::SiteWithout))
|| matches!(field, Field::Text(TextField::Domain)))
}
@@ -215,7 +215,7 @@ impl FastSiteDomainPatternWeight {
let fieldnorm_reader = self.fieldnorm_reader(reader)?;
let field_no_tokenizer = match ALL_FIELDS[self.field.field_id() as usize] {
- Field::Text(TextField::Site) => Field::Text(TextField::SiteNoTokenizer),
+ Field::Text(TextField::SiteWithout) => Field::Text(TextField::SiteNoTokenizer),
Field::Text(TextField::Domain) => Field::Text(TextField::DomainNoTokenizer),
_ => unreachable!(),
};
@@ -320,7 +320,7 @@ impl PatternWeight {
Field::Text(TextField::CleanBody) => Ok(FastField::NumCleanBodyTokens),
Field::Text(TextField::Url) => Ok(FastField::NumUrlTokens),
Field::Text(TextField::Domain) => Ok(FastField::NumDomainTokens),
- Field::Text(TextField::Site) => Ok(FastField::NumSiteTokens),
+ Field::Text(TextField::SiteWithout) => Ok(FastField::NumSiteTokens),
Field::Text(TextField::Description) => Ok(FastField::NumDescriptionTokens),
Field::Text(TextField::FlattenedSchemaOrgJson) => {
Ok(FastField::NumFlattenedSchemaTokens)
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index 67f679bc..606a1e5b 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -347,7 +347,8 @@ mod tests {
rand_words(1000)
),
"https://e.com",
- ),
+ )
+ .unwrap(),
backlinks: vec![],
host_centrality: 0.0,
fetch_time_ms: 500,
@@ -375,7 +376,8 @@ mod tests {
rand_words(1000)
),
"https://d.com",
- ),
+ )
+ .unwrap(),
backlinks: vec![],
host_centrality: 0.01,
fetch_time_ms: 500,
@@ -406,7 +408,7 @@ mod tests {
.webpages;
assert_eq!(res.len(), 2);
- assert_eq!(&res[0].url, "https://e.com");
- assert_eq!(&res[1].url, "https://d.com");
+ assert_eq!(&res[0].url, "https://e.com/");
+ assert_eq!(&res[1].url, "https://d.com/");
}
}
diff --git a/core/src/ranking/mod.rs b/core/src/ranking/mod.rs
index bae1dbc7..b775bb77 100644
--- a/core/src/ranking/mod.rs
+++ b/core/src/ranking/mod.rs
@@ -135,6 +135,7 @@ impl Ranker {
mod tests {
use optics::Optic;
+ use url::Url;
use crate::{
index::Index,
@@ -167,7 +168,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.a.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
fetch_time_ms: 500,
@@ -195,7 +196,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.b.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 5.0,
fetch_time_ms: 500,
@@ -216,8 +217,8 @@ mod tests {
})
.expect("Search failed");
assert_eq!(result.webpages.len(), 2);
- assert_eq!(result.webpages[0].url, "https://www.b.com");
- assert_eq!(result.webpages[1].url, "https://www.a.com");
+ assert_eq!(result.webpages[0].url, "https://www.b.com/");
+ assert_eq!(result.webpages[1].url, "https://www.a.com/");
}
#[test]
@@ -242,7 +243,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.a.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
fetch_time_ms: 500,
@@ -270,7 +271,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.b.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
fetch_time_ms: 500,
@@ -291,8 +292,8 @@ mod tests {
})
.expect("Search failed");
assert_eq!(result.webpages.len(), 2);
- assert_eq!(result.webpages[0].url, "https://www.b.com");
- assert_eq!(result.webpages[1].url, "https://www.a.com");
+ assert_eq!(result.webpages[0].url, "https://www.b.com/");
+ assert_eq!(result.webpages[1].url, "https://www.a.com/");
}
#[test]
@@ -316,7 +317,7 @@ mod tests {
"#
),
"https://www.old.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 499,
@@ -344,7 +345,7 @@ mod tests {
"#
),
"https://www.new.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 500,
@@ -365,7 +366,7 @@ mod tests {
})
.expect("Search failed");
- assert_eq!(result.webpages[0].url, "https://www.new.com");
+ assert_eq!(result.webpages[0].url, "https://www.new.com/");
}
#[test]
@@ -386,7 +387,7 @@ mod tests {
</html>
"#,
"https://www.first.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.0,
fetch_time_ms: 500,
@@ -423,7 +424,7 @@ mod tests {
</body>
</html>
"#,
- "https://www.second.com"),
+ "https://www.second.com").unwrap(),
backlinks: vec![],
host_centrality: 0.00003,
fetch_time_ms: 500,
@@ -445,7 +446,7 @@ mod tests {
.expect("Search failed");
assert_eq!(result.webpages.len(), 2);
- assert_eq!(result.webpages[0].url, "https://www.first.com");
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
}
#[test]
@@ -466,10 +467,10 @@ mod tests {
</html>
"#,
"https://www.first.com",
- ),
+ ).unwrap(),
backlinks: vec![Link {
- source: "https://www.second.com".to_string().into(),
- destination: "https://www.first.com".to_string().into(),
+ source: Url::parse("https://www.second.com").unwrap(),
+ destination: Url::parse("https://www.first.com").unwrap(),
text: "test this is the best test site".to_string(),
}],
host_centrality: 0.0,
@@ -495,7 +496,7 @@ mod tests {
</html>
"#,
"https://www.second.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 0.00003,
fetch_time_ms: 500,
@@ -517,7 +518,7 @@ mod tests {
.expect("Search failed");
assert_eq!(result.webpages.len(), 2);
- assert_eq!(result.webpages[0].url, "https://www.first.com");
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
}
#[test]
@@ -538,7 +539,7 @@ mod tests {
</html>
"#,
"https://www.body.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 20,
@@ -564,7 +565,7 @@ mod tests {
</html>
"#,
"https://www.title.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 20,
@@ -591,7 +592,7 @@ mod tests {
</html>
"#,
"https://www.centrality.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 1.02,
fetch_time_ms: 500,
@@ -624,7 +625,7 @@ mod tests {
.unwrap();
assert_eq!(res.num_hits, 3);
- assert_eq!(&res.webpages[0].url, "https://www.title.com");
+ assert_eq!(&res.webpages[0].url, "https://www.title.com/");
let res = searcher
.search(&SearchQuery {
@@ -642,7 +643,7 @@ mod tests {
.unwrap();
assert_eq!(res.num_hits, 3);
- assert_eq!(&res.webpages[0].url, "https://www.centrality.com");
+ assert_eq!(&res.webpages[0].url, "https://www.centrality.com/");
}
#[test]
@@ -664,7 +665,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.first.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 1.0,
pre_computed_score: 0.0,
@@ -696,7 +697,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.third.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 500,
@@ -728,7 +729,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.second.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 500,
@@ -756,9 +757,9 @@ mod tests {
assert_eq!(result.num_hits, 3);
assert_eq!(result.webpages.len(), 3);
- assert_eq!(result.webpages[0].url, "https://www.first.com");
- assert_eq!(result.webpages[1].url, "https://www.second.com");
- assert_eq!(result.webpages[2].url, "https://www.third.com");
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
+ assert_eq!(result.webpages[1].url, "https://www.second.com/");
+ assert_eq!(result.webpages[2].url, "https://www.third.com/");
}
#[test]
@@ -782,7 +783,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.first.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 0,
@@ -810,7 +811,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.second.com",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 5000,
@@ -833,8 +834,8 @@ mod tests {
assert_eq!(result.num_hits, 2);
assert_eq!(result.webpages.len(), 2);
- assert_eq!(result.webpages[0].url, "https://www.first.com");
- assert_eq!(result.webpages[1].url, "https://www.second.com");
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
+ assert_eq!(result.webpages[1].url, "https://www.second.com/");
}
@@ -859,7 +860,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.first.com/one",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 2,
@@ -887,7 +888,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.second.com/one/two",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 1,
@@ -915,7 +916,7 @@ mod tests {
crate::rand_words(100)
),
"https://www.third.com/one/two123",
- ),
+ ).unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 0,
diff --git a/core/src/ranking/optics.rs b/core/src/ranking/optics.rs
index c107bfc9..2ac97ecc 100644
--- a/core/src/ranking/optics.rs
+++ b/core/src/ranking/optics.rs
@@ -94,7 +94,8 @@ mod tests {
crate::rand_words(100)
),
"https://www.first.com",
- ),
+ )
+ .unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 50,
@@ -122,7 +123,8 @@ mod tests {
crate::rand_words(100)
),
"https://www.second.com",
- ),
+ )
+ .unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 49,
@@ -150,7 +152,8 @@ mod tests {
crate::rand_words(100)
),
"https://www.third.com",
- ),
+ )
+ .unwrap(),
backlinks: vec![],
host_centrality: 1.0,
fetch_time_ms: 0,
@@ -180,9 +183,9 @@ mod tests {
assert_eq!(result.num_hits, 3);
assert_eq!(result.webpages.len(), 3);
- assert_eq!(result.webpages[0].url, "https://www.first.com");
- assert_eq!(result.webpages[1].url, "https://www.second.com");
- assert_eq!(result.webpages[2].url, "https://www.third.com");
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
+ assert_eq!(result.webpages[1].url, "https://www.second.com/");
+ assert_eq!(result.webpages[2].url, "https://www.third.com/");
let result = searcher
.search(&SearchQuery {
@@ -198,9 +201,9 @@ mod tests {
assert_eq!(result.num_hits, 3);
assert_eq!(result.webpages.len(), 3);
- assert_eq!(result.webpages[0].url, "https://www.first.com");
- assert_eq!(result.webpages[1].url, "https://www.second.com");
- assert_eq!(result.webpages[2].url, "https://www.third.com");
+ assert_eq!(result.webpages[0].url, "https://www.first.com/");
+ assert_eq!(result.webpages[1].url, "https://www.second.com/");
+ assert_eq!(result.webpages[2].url, "https://www.third.com/");
let result = searcher
.search(&SearchQuery {
@@ -217,7 +220,7 @@ mod tests {
assert_eq!(result.num_hits, 2);
assert_eq!(result.webpages.len(), 2);
- assert_eq!(result.webpages[0].url, "https://www.third.com");
- assert_eq!(result.webpages[1].url, "https://www.second.com");
+ assert_eq!(result.webpages[0].url, "https://www.third.com/");
+ assert_eq!(result.webpages[1].url, "https://www.second.com/");
}
}
diff --git a/core/src/ranking/signal.rs b/core/src/ranking/signal.rs
index a22e443b..1d5ea5f5 100644
--- a/core/src/ranking/signal.rs
+++ b/core/src/ranking/signal.rs
@@ -484,7 +484,7 @@ impl Signal {
let value = match self {
Signal::HostCentrality => Some(webpage.host_centrality),
Signal::PageCentrality => Some(webpage.page_centrality),
- Signal::IsHomepage => Some(webpage.html.url().is_homepage().into()),
+ Signal::IsHomepage => Some(webpage.html.is_homepage().into()),
Signal::FetchTimeMs => {
let fetch_time_ms = webpage.fetch_time_ms as usize;
if fetch_time_ms >= signal_aggregator.fetch_time_ms_cache.len() {
@@ -511,20 +511,28 @@ impl Signal {
Some(score_region(region, signal_aggregator))
}
Signal::UrlDigits => {
- let num_digits = webpage
+ let num_digits = (webpage
.html
.url()
- .path_and_query()
+ .path()
.chars()
.filter(|c| c.is_ascii_digit())
- .count() as f64;
+ .count()
+ + webpage
+ .html
+ .url()
+ .query()
+ .unwrap_or_default()
+ .chars()
+ .filter(|c| c.is_ascii_digit())
+ .count()) as f64;
Some(score_digits(num_digits))
}
Signal::UrlSlashes => {
let num_slashes = webpage
.html
.url()
- .path_and_query()
+ .path()
.chars()
.filter(|c| c == &'/')
.count() as f64;
@@ -610,7 +618,7 @@ impl Signal {
Signal::Bm25StemmedCleanBody => Some(TextField::StemmedCleanBody),
Signal::Bm25AllBody => Some(TextField::AllBody),
Signal::Bm25Url => Some(TextField::Url),
- Signal::Bm25Site => Some(TextField::Site),
+ Signal::Bm25Site => Some(TextField::SiteWithout),
Signal::Bm25Domain => Some(TextField::Domain),
Signal::Bm25SiteNoTokenizer => Some(TextField::SiteNoTokenizer),
Signal::Bm25DomainNoTokenizer => Some(TextField::DomainNoTokenizer),
@@ -1116,7 +1124,8 @@ mod tests {
crate::rand_words(1_000)
),
"https://www.example.com",
- );
+ )
+ .unwrap();
webpage
.html
diff --git a/core/src/schema.rs b/core/src/schema.rs
index 766f4cb5..02a7e9f5 100644
--- a/core/src/schema.rs
+++ b/core/src/schema.rs
@@ -31,7 +31,7 @@ pub enum TextField {
AllBody,
Url,
UrlNoTokenizer,
- Site,
+ SiteWithout,
Domain,
SiteNoTokenizer,
SiteIfHomepageNoTokenizer,
@@ -69,7 +69,7 @@ impl TextField {
TextField::AllBody => Tokenizer::default(),
TextField::Url => Tokenizer::default(),
TextField::UrlNoTokenizer => Tokenizer::Identity(Identity {}),
- TextField::Site => Tokenizer::default(),
+ TextField::SiteWithout => Tokenizer::default(),
TextField::Domain => Tokenizer::default(),
TextField::SiteNoTokenizer => Tokenizer::Identity(Identity {}),
TextField::SiteIfHomepageNoTokenizer => Tokenizer::Identity(Identity {}),
@@ -106,7 +106,7 @@ impl TextField {
TextField::AllBody => false,
TextField::Url => true,
TextField::UrlNoTokenizer => false,
- TextField::Site => true,
+ TextField::SiteWithout => true,
TextField::Domain => true,
TextField::SiteNoTokenizer => false,
TextField::SiteIfHomepageNoTokenizer => false,
@@ -132,7 +132,7 @@ impl TextField {
TextField::CleanBody => "body",
TextField::Url => "url",
TextField::UrlNoTokenizer => "url_no_tokenizer",
- TextField::Site => "site",
+ TextField::SiteWithout => "site",
TextField::Domain => "domain",
TextField::SiteNoTokenizer => "site_no_tokenizer",
TextField::SiteIfHomepageNoTokenizer => "site_if_homepage_no_tokenizer",
@@ -247,7 +247,7 @@ pub static ALL_FIELDS: [Field; 54] = [
Field::Text(TextField::AllBody),
Field::Text(TextField::Url),
Field::Text(TextField::UrlNoTokenizer),
- Field::Text(TextField::Site),
+ Field::Text(TextField::SiteWithout),
Field::Text(TextField::Domain),
Field::Text(TextField::SiteNoTokenizer),
Field::Text(TextField::SiteIfHomepageNoTokenizer),
@@ -328,7 +328,9 @@ impl Field {
Field::Text(TextField::UrlNoTokenizer) => {
IndexingOption::Text(self.default_text_options())
}
- Field::Text(TextField::Site) => IndexingOption::Text(self.default_text_options()),
+ Field::Text(TextField::SiteWithout) => {
+ IndexingOption::Text(self.default_text_options())
+ }
Field::Text(TextField::SiteIfHomepageNoTokenizer) => {
IndexingOption::Text(self.default_text_options())
}
diff --git a/core/src/search_prettifier/mod.rs b/core/src/search_prettifier/mod.rs
index b5eb0f56..c530e7a0 100644
--- a/core/src/search_prettifier/mod.rs
+++ b/core/src/search_prettifier/mod.rs
@@ -22,12 +22,12 @@ use std::collections::HashMap;
use chrono::{NaiveDateTime, Utc};
use itertools::{intersperse, Itertools};
use serde::{Deserialize, Serialize};
+use url::Url;
use crate::{
inverted_index::RetrievedWebpage,
ranking::Signal,
spell::{self, CorrectionTerm},
- webpage::Url,
};
pub use self::stack_overflow::{create_stackoverflow_sidebar, CodeOrText};
@@ -90,17 +90,23 @@ pub fn html_escape(s: &str) -> String {
}
fn prettify_url(url: &Url) -> String {
- let mut pretty_url = url.strip_query().to_string();
+ let mut pretty_url = url.clone();
+ pretty_url.set_query(None);
+
+ let scheme = pretty_url.scheme().to_string();
+
+ let mut pretty_url = pretty_url.to_string();
+
+ if let Some(stripped) = pretty_url.strip_prefix((scheme.clone() + "://").as_str()) {
+ pretty_url = stripped.to_string();
+ }
if pretty_url.ends_with('/') {
pretty_url = pretty_url.chars().take(pretty_url.len() - 1).collect();
}
- let protocol = Url::from(pretty_url.clone()).protocol().to_string() + "://";
- pretty_url = Url::from(pretty_url.clone())
- .strip_protocol()
- .replace('/', " › ");
- pretty_url = protocol + pretty_url.as_str();
+ pretty_url = pretty_url.replace('/', " › ");
+ pretty_url = scheme + "://" + pretty_url.as_str();
pretty_url
}
@@ -133,9 +139,9 @@ fn prettify_date(date: NaiveDateTime) -> String {
fn generate_snippet(webpage: &RetrievedWebpage) -> Snippet {
let last_updated = webpage.updated_time.map(prettify_date);
- let url = Url::from(webpage.url.clone());
+ let url = Url::parse(&webpage.url).unwrap();
- if url.domain() == "stackoverflow.com"
+ if url.domain().unwrap_or_default() == "stackoverflow.com"
&& webpage
.schema_org
.iter()
@@ -178,15 +184,15 @@ impl From<RetrievedWebpage> for DisplayedWebpage {
fn from(webpage: RetrievedWebpage) -> Self {
let snippet = generate_snippet(&webpage);
- let url: Url = webpage.url.clone().into();
- let domain = url.domain().to_string();
+ let url = Url::parse(&webpage.url).unwrap();
+ let domain = url.domain().unwrap_or_default().to_string();
let pretty_url = prettify_url(&url);
let title = html_escape(&webpage.title);
Self {
title,
- site: url.site().to_string(),
+ site: url.host_str().unwrap_or_default().to_string(),
url: webpage.url,
pretty_url,
domain,
diff --git a/core/src/search_prettifier/stack_overflow.rs b/core/src/search_prettifier/stack_overflow.rs
index 516d153c..deb0dc90 100644
--- a/core/src/search_prettifier/stack_overflow.rs
+++ b/core/src/search_prettifier/stack_overflow.rs
@@ -15,13 +15,11 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use serde::{Deserialize, Serialize};
+use url::Url;
use crate::{
inverted_index::RetrievedWebpage,
- webpage::{
- schema_org::{self, Item, OneOrMany, Property},
- Url,
- },
+ webpage::schema_org::{self, Item, OneOrMany, Property},
Error,
};
@@ -90,17 +88,13 @@ fn parse_so_answer(
let url = url
.one()
.and_then(|prop| prop.try_into_string())
- .map(Url::from)
- .map(|mut url| {
- url = url.into_absolute(&Url::from(webpage_url.site()));
- url
- })?;
+ .and_then(|s| Url::parse(&s).or_else(|_| webpage_url.join(&s)).ok())?;
Some(StackOverflowAnswer {
body: text,
date: format!("{}", date.date().format("%b %d, %Y")),
upvotes,
- url: url.full(),
+ url: url.to_string(),
accepted,
})
}
@@ -165,7 +159,11 @@ pub fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {
.and_then(|ans| ans.one())
.and_then(|prop| prop.try_into_item())
.and_then(|item| {
- schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), true)
+ schema_item_to_stackoverflow_answer(
+ item,
+ Url::parse(&webpage.url).unwrap(),
+ true,
+ )
})
{
answers.push(ans);
@@ -180,7 +178,11 @@ pub fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {
.into_iter()
.filter_map(|prop| prop.try_into_item())
.filter_map(|item| {
- schema_item_to_stackoverflow_answer(item, Url::from(webpage.url.clone()), false)
+ schema_item_to_stackoverflow_answer(
+ item,
+ Url::parse(&webpage.url).unwrap(),
+ false,
+ )
})
{
answers.push(answer);
@@ -191,11 +193,11 @@ pub fn stackoverflow_snippet(webpage: &RetrievedWebpage) -> Result<Snippet> {
answers: answers.into_iter().take(3).collect(),
})
}
- None => Err(Error::InvalidStackoverflowSchema),
+ None => Err(Error::InvalidStackoverflowSchema.into()),
}
}
-pub fn create_stackoverflow_sidebar(schema_org: Vec<Item>, url: String) -> Result<Sidebar> {
+pub fn create_stackoverflow_sidebar(schema_org: Vec<Item>, url: Url) -> Result<Sidebar> {
if let Some(item) = schema_org
.into_iter()
.find(|item| item.types_contains("QAPage"))
@@ -216,12 +218,10 @@ pub fn create_stackoverflow_sidebar(schema_org: Vec<Item>, url: String) -> Resul
.cloned()
.and_then(|ans| ans.one())
.and_then(|prop| prop.try_into_item())
- .and_then(|item| {
- schema_item_to_stackoverflow_answer(item, Url::from(url.clone()), true)
- })
+ .and_then(|item| schema_item_to_stackoverflow_answer(item, url, true))
.map(|answer| Sidebar::StackOverflow { title, answer })
- .ok_or(Error::InvalidStackoverflowSchema)
+ .ok_or(Error::InvalidStackoverflowSchema.into())
} else {
- Err(Error::InvalidStackoverflowSchema)
+ Err(Error::InvalidStackoverflowSchema.into())
}
}
diff --git a/core/src/searcher/distributed.rs b/core/src/searcher/distributed.rs
index 9a3f4d82..24fcef4c 100644
--- a/core/src/searcher/distributed.rs
+++ b/core/src/searcher/distributed.rs
@@ -18,7 +18,6 @@ use crate::{
distributed::{cluster::Cluster, member::Service, retry_strategy::ExponentialBackoff},
inverted_index::{self, RetrievedWebpage},
ranking::pipeline::{AsRankingWebsite, RankingWebsite},
- webpage::Url,
Result,
};
@@ -29,6 +28,7 @@ use futures::StreamExt;
use rand::seq::SliceRandom;
use serde::{Deserialize, Serialize};
use thiserror::Error;
+use url::Url;
use crate::distributed::sonic;
@@ -106,7 +106,7 @@ impl RemoteSearcher {
Err(Error::WebpageNotFound.into())
}
- async fn get_homepage_descriptions(&self, urls: &[String]) -> HashMap<Url, String> {
+ async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {
let mut conn = self.conn();
if let Ok(sonic::Response::Content(body)) = conn
@@ -202,7 +202,7 @@ impl Shard {
Err(Error::SearchFailed.into())
}
- async fn get_homepage_descriptions(&self, urls: &[String]) -> HashMap<Url, String> {
+ async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {
self.replica().get_homepage_descriptions(urls).await
}
}
@@ -224,7 +224,7 @@ pub enum Request {
url: String,
},
GetHomepageDescriptions {
- urls: Vec<String>,
+ urls: Vec<Url>,
},
}
@@ -335,7 +335,7 @@ impl DistributedSearcher {
.ok_or(Error::WebpageNotFound.into())
}
- pub async fn get_homepage_descriptions(&self, urls: &[String]) -> HashMap<Url, String> {
+ pub async fn get_homepage_descriptions(&self, urls: &[Url]) -> HashMap<Url, String> {
self.shards()
.await
.iter()
diff --git a/core/src/searcher/frontend.rs b/core/src/searcher/frontend.rs
index 53c80460..51c5b09f 100644
--- a/core/src/searcher/frontend.rs
+++ b/core/src/searcher/frontend.rs
@@ -22,6 +22,7 @@ use std::time::Instant;
use itertools::intersperse;
use optics::Optic;
+use url::Url;
use crate::bangs::{Bang, BangHit};
use crate::config::{CollectorConfig, FrontendThresholds};
@@ -149,7 +150,10 @@ impl FrontendSearcher {
.await;
if let Some(res) = retrieved.pop() {
- return Ok(Some(create_stackoverflow_sidebar(res.schema_org, res.url)?));
+ return Ok(Some(create_stackoverflow_sidebar(
+ res.schema_org,
+ Url::parse(&res.url).unwrap(),
+ )?));
}
}
}
@@ -209,7 +213,7 @@ impl FrontendSearcher {
tag: String::new(),
url: webpage.url.clone(),
},
- redirect_to: webpage.url.clone().into(),
+ redirect_to: Url::parse(&webpage.url).unwrap(),
}));
}
@@ -385,7 +389,7 @@ impl FrontendSearcher {
pub async fn search(&self, query: &SearchQuery) -> Result<SearchResult> {
if let Some(bang) = self.check_bangs(query).await? {
- return Ok(SearchResult::Bang(bang));
+ return Ok(SearchResult::Bang(Box::new(bang)));
}
Ok(SearchResult::Websites(self.search_websites(query).await?))
diff --git a/core/src/searcher/local.rs b/core/src/searcher/local.rs
index 8f66b799..3390e702 100644
--- a/core/src/searcher/local.rs
+++ b/core/src/searcher/local.rs
@@ -17,6 +17,8 @@
use std::collections::HashMap;
use std::sync::Arc;
+use url::Url;
+
use crate::config::{CollectorConfig, SnippetConfig};
use crate::entity_index::{EntityIndex, EntityMatch};
use crate::image_store::Image;
@@ -103,7 +105,7 @@ impl LocalSearcher {
let parsed_query = Query::parse(ctx, query, &self.index.inverted_index)?;
if parsed_query.is_empty() {
- Err(Error::EmptyQuery)
+ Err(Error::EmptyQuery.into())
} else {
Ok(parsed_query)
}
@@ -294,7 +296,7 @@ impl LocalSearcher {
let query = Query::parse(&ctx, &query, &self.index.inverted_index)?;
if query.is_empty() {
- return Err(Error::EmptyQuery);
+ return Err(Error::EmptyQuery.into());
}
self.index.retrieve_websites(websites, &query)
@@ -388,7 +390,7 @@ impl LocalSearcher {
self.index.get_webpage(url)
}
- pub fn get_homepage(&self, url: &str) -> Option<RetrievedWebpage> {
+ pub fn get_homepage(&self, url: &Url) -> Option<RetrievedWebpage> {
self.index.get_homepage(url)
}
}
@@ -424,7 +426,8 @@ mod tests {
</html>
"#,
&format!("https://www.{i}.com"),
- ),
+ )
+ .unwrap(),
backlinks: vec![],
host_centrality: (NUM_WEBSITES - i) as f64,
fetch_time_ms: 500,
@@ -458,7 +461,7 @@ mod tests {
for (i, url) in urls.into_iter().enumerate() {
assert_eq!(
url,
- format!("https://www.{}.com", i + (p * NUM_RESULTS_PER_PAGE))
+ format!("https://www.{}.com/", i + (p * NUM_RESULTS_PER_PAGE))
)
}
}
@@ -482,7 +485,7 @@ mod tests {
"#
,
"https://www.example.com",
- ))
+ ).unwrap())
.expect("failed to insert webpage");
index.commit().unwrap();
diff --git a/core/src/searcher/mod.rs b/core/src/searcher/mod.rs
index a16e80ed..8d65a7e9 100644
--- a/core/src/searcher/mod.rs
+++ b/core/src/searcher/mod.rs
@@ -39,7 +39,7 @@ pub const NUM_RESULTS_PER_PAGE: usize = 20;
#[derive(Debug, Serialize, Deserialize)]
pub enum SearchResult {
Websites(WebsitesResult),
- Bang(BangHit),
+ Bang(Box<BangHit>),
}
#[derive(Debug, Serialize, Deserialize)]
diff --git a/core/src/snippet.rs b/core/src/snippet.rs
index 8a7d5426..db2523a9 100644
--- a/core/src/snippet.rs
+++ b/core/src/snippet.rs
@@ -270,9 +270,10 @@ Survey in 2016, 2017, and 2018."#;
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Website for runners</title>
@@ -282,9 +283,11 @@ Survey in 2016, 2017, and 2018."#;
</body>
</html>
"#
- ),
- "https://www.example.com",
- ))
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -307,9 +310,10 @@ Survey in 2016, 2017, and 2018."#;
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Website for runners</title>
@@ -319,9 +323,11 @@ Survey in 2016, 2017, and 2018."#;
</body>
</html>
"#
- ),
- "https://www.example.com",
- ))
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
@@ -344,9 +350,10 @@ Survey in 2016, 2017, and 2018."#;
let mut index = Index::temporary().expect("Unable to open index");
index
- .insert(Webpage::new(
- &format!(
- r#"
+ .insert(
+ Webpage::new(
+ &format!(
+ r#"
<html>
<head>
<title>Website for runners</title>
@@ -356,9 +363,11 @@ Survey in 2016, 2017, and 2018."#;
</body>
</html>
"#
- ),
- "https://www.example.com",
- ))
+ ),
+ "https://www.example.com",
+ )
+ .unwrap(),
+ )
.expect("failed to insert webpage");
index.commit().expect("failed to commit index");
diff --git a/core/src/subdomain_count.rs b/core/src/subdomain_count.rs
index 66bc008e..8a0edd1d 100644
--- a/core/src/subdomain_count.rs
+++ b/core/src/subdomain_count.rs
@@ -15,11 +15,12 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use crate::{
+ ceil_char_boundary,
kv::{rocksdb_store::RocksDbStore, Kv},
prehashed::{hash, Prehashed},
- webpage::Url,
};
use std::{collections::HashSet, path::Path};
+use url::Url;
pub struct SubdomainCounter {
inner: Box<dyn Kv<Prehashed, HashSet<String>>>,
@@ -33,12 +34,22 @@ impl SubdomainCounter {
}
pub fn increment(&mut self, url: Url) {
- if let Some(subdomain) = url.subdomain() {
- let domain = hash(url.domain());
+ let scheme = url.scheme();
+ let domain = url.domain().unwrap_or_default();
+
+ let mut url = url.as_str().strip_prefix(scheme).unwrap_or_default();
+
+ if let Some(slash) = url.find('/') {
+ url = &url[..ceil_char_boundary(url, slash)];
+ }
+
+ if let Some(subdomain) = url.strip_suffix(domain) {
+ let domain = hash(domain);
let subdomain = subdomain.to_string();
let mut set = self.inner.get(&domain).unwrap_or_default();
set.insert(subdomain);
+
self.inner.insert(domain, set);
}
}
diff --git a/core/src/warc.rs b/core/src/warc.rs
index 5e882d34..fccde2c7 100644
--- a/core/src/warc.rs
+++ b/core/src/warc.rs
@@ -123,7 +123,7 @@ impl WarcFile {
sleep(dur);
}
- Err(Error::DownloadFailed)
+ Err(Error::DownloadFailed.into())
}
fn load_from_folder<W: Write + Seek>(name: &str, folder: &str, buf: &mut W) -> Result<()> {
@@ -155,7 +155,7 @@ impl WarcFile {
let res = client.get(url).send()?;
if res.status().as_u16() != 200 {
- return Err(Error::DownloadFailed);
+ return Err(Error::DownloadFailed.into());
}
let bytes = res.bytes()?;
@@ -271,7 +271,7 @@ impl Metadata {
}
}
- Err(Error::WarcParse("Failed to parse metadata"))
+ Err(Error::WarcParse("Failed to parse metadata").into())
}
}
@@ -295,7 +295,7 @@ impl<R: Read> RecordIterator<R> {
rtrim(&mut version);
if !version.to_uppercase().starts_with("WARC/1.") {
- return Some(Err(Error::WarcParse("Unknown WARC version")));
+ return Some(Err(Error::WarcParse("Unknown WARC version").into()));
}
let mut header = BTreeMap::<String, String>::new();
@@ -303,7 +303,7 @@ impl<R: Read> RecordIterator<R> {
loop {
let mut line_buf = String::new();
if let Err(io) = self.reader.read_line(&mut line_buf) {
- return Some(Err(Error::IOError(io)));
+ return Some(Err(io.into()));
}
rtrim(&mut line_buf);
@@ -321,33 +321,36 @@ impl<R: Read> RecordIterator<R> {
} else {
return Some(Err(Error::WarcParse(
"All header lines must contain a colon",
- )));
+ )
+ .into()));
}
}
let content_len = header.get("CONTENT-LENGTH");
if content_len.is_none() {
- return Some(Err(Error::WarcParse("Record has no content-length")));
+ return Some(Err(Error::WarcParse("Record has no content-length").into()));
}
let content_len = content_len.unwrap().parse::<usize>();
if content_len.is_err() {
- return Some(Err(Error::WarcParse("Could not parse content length")));
+ return Some(Err(
+ Error::WarcParse("Could not parse content length").into()
+ ));
}
let content_len = content_len.unwrap();
let mut content = vec![0; content_len];
if let Err(io) = self.reader.read_exact(&mut content) {
- return Some(Err(Error::IOError(io)));
+ return Some(Err(io.into()));
}
let mut linefeed = [0u8; 4];
if let Err(io) = self.reader.read_exact(&mut linefeed) {
- return Some(Err(Error::IOError(io)));
+ return Some(Err(io.into()));
}
if linefeed != [13, 10, 13, 10] {
- return Some(Err(Error::WarcParse("Invalid record ending")));
+ return Some(Err(Error::WarcParse("Invalid record ending").into()));
}
let record = RawWarcRecord { header, content };
@@ -386,7 +389,7 @@ impl<R: Read> Iterator for RecordIterator<R> {
"response" => response = Some(Response::from_raw(item)),
"metadata" => metadata = Some(Metadata::from_raw(item)),
_ => {
- return Some(Err(Error::WarcParse("Unsupported WARC type")));
+ return Some(Err(Error::WarcParse("Unsupported WARC type").into()));
}
}
}
@@ -395,7 +398,8 @@ impl<R: Read> Iterator for RecordIterator<R> {
if request.is_none() || response.is_none() || metadata.is_none() {
return Some(Err(Error::WarcParse(
"Request, response or metadata not found",
- )));
+ )
+ .into()));
}
let request = request.unwrap();
@@ -405,7 +409,8 @@ impl<R: Read> Iterator for RecordIterator<R> {
if request.is_err() || response.is_err() || metadata.is_err() {
return Some(Err(Error::WarcParse(
"Request, response or metadata is error",
- )));
+ )
+ .into()));
}
let request = request.unwrap();
@@ -494,7 +499,7 @@ impl WarcWriter {
writer.write_all("\r\n\r\n".as_bytes())?;
writer.flush().unwrap();
- let bytes = writer.finish().map_err(Error::IOError)?;
+ let bytes = writer.finish()?;
self.buf.extend_from_slice(&bytes);
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index 5d7c4e94..3796cd31 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -15,8 +15,6 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
mod segment;
-use lru::LruCache;
-use serde::{Deserialize, Serialize};
use std::collections::{BTreeMap, BinaryHeap, HashSet};
use std::fs::File;
use std::io::{BufReader, BufWriter, Read, Write};
@@ -24,10 +22,13 @@ use std::path::Path;
use std::sync::{Arc, Mutex};
use std::{cmp, fs};
+use lru::LruCache;
+use serde::{Deserialize, Serialize};
+use url::Url;
+
use crate::directory::{self, DirEntry};
use crate::executor::Executor;
use crate::intmap;
-use crate::webpage::Url;
pub mod centrality;
mod store;
@@ -91,12 +92,20 @@ pub struct Node {
impl Node {
pub fn into_host(self) -> Node {
- let url = Url::from(self.name);
-
- let host = url.host_without_www();
+ let url = if self.name.contains("://") {
+ Url::parse(&self.name)
+ } else {
+ Url::parse(&("http://".to_string() + self.name.as_str()))
+ };
- Node {
- name: host.to_string(),
+ match url {
+ Ok(url) => {
+ let host = url.host_str().unwrap_or_default().to_string();
+ Node { name: host }
+ }
+ Err(_) => Node {
+ name: String::new(),
+ },
}
}
@@ -108,17 +117,20 @@ impl Node {
impl From<String> for Node {
fn from(name: String) -> Self {
- let url = Url::from(name);
- url.into()
+ let url = if name.contains("://") {
+ Url::parse(&name).unwrap()
+ } else {
+ Url::parse(&("http://".to_string() + name.as_str())).unwrap()
+ };
+
+ Node::from(&url)
}
}
impl From<&Url> for Node {
fn from(url: &Url) -> Self {
- let normalized = url.normalize();
- Node {
- name: normalized.into(),
- }
+ let normalized = normalize_url(url);
+ Node { name: normalized }
}
}
@@ -134,6 +146,52 @@ impl From<Url> for Node {
}
}
+pub fn normalize_url(url: &Url) -> String {
+ let mut url = url.clone();
+ let allowed_queries: Vec<_> = url
+ .query_pairs()
+ .filter(|(key, _)| {
+ !key.starts_with("utm_")
+ && !key.starts_with("fbclid")
+ && !key.starts_with("gclid")
+ && !key.starts_with("msclkid")
+ })
+ .map(|(key, value)| (key.to_string(), value.to_string()))
+ .collect();
+
+ {
+ let mut queries = url.query_pairs_mut();
+ queries.clear();
+
+ if !allowed_queries.is_empty() {
+ queries.extend_pairs(allowed_queries);
+ }
+ }
+
+ if url.query().unwrap_or_default().is_empty() {
+ url.set_query(None);
+ }
+
+ let scheme = url.scheme();
+ let mut normalized = url
+ .as_str()
+ .strip_prefix(scheme)
+ .unwrap_or_default()
+ .strip_prefix("://")
+ .unwrap_or_default()
+ .to_string();
+
+ if let Some(stripped) = normalized.strip_prefix("www.") {
+ normalized = stripped.to_string();
+ }
+
+ if let Some(prefix) = normalized.strip_suffix('/') {
+ normalized = prefix.to_string();
+ }
+
+ normalized
+}
+
#[derive(Debug, PartialEq, Eq)]
pub struct Edge {
pub from: NodeID,
@@ -746,9 +804,9 @@ mod test {
#[test]
fn remove_protocol() {
- let n = Node::from("https://www.example.com?test");
+ let n = Node::from("https://www.example.com/?test");
- assert_eq!(&n.name, "example.com?test");
+ assert_eq!(&n.name, "example.com/?test=");
}
#[test]
diff --git a/core/src/webpage/mod.rs b/core/src/webpage/mod.rs
index ee111c29..048df268 100644
--- a/core/src/webpage/mod.rs
+++ b/core/src/webpage/mod.rs
@@ -27,16 +27,15 @@ use kuchiki::{iter::NodeEdge, traits::TendrilSink, NodeRef};
use regex::Regex;
use std::{collections::HashMap, panic, str::FromStr};
use tantivy::tokenizer::{PreTokenizedString, Tokenizer};
+use url::Url;
use whatlang::Lang;
mod just_text;
pub mod region;
pub mod schema_org;
-mod url;
use crate::schema::{Field, ALL_FIELDS, FLOAT_SCALING};
-pub use self::url::Url;
use self::{
just_text::{JustText, Paragraph},
region::Region,
@@ -123,10 +122,10 @@ pub struct Webpage {
impl Webpage {
#[cfg(test)]
- pub fn new(html: &str, url: &str) -> Self {
- let html = Html::parse(html, url);
+ pub fn new(html: &str, url: &str) -> Result<Self> {
+ let html = Html::parse(html, url)?;
- Self {
+ Ok(Self {
html,
backlinks: Vec::new(),
host_centrality: 0.0,
@@ -135,7 +134,7 @@ impl Webpage {
pre_computed_score: 0.0,
node_id: None,
dmoz_description: None,
- }
+ })
}
fn dmoz_description(&self) -> Option<String> {
@@ -273,13 +272,13 @@ enum RobotsMeta {
}
impl FromStr for RobotsMeta {
- type Err = Error;
+ type Err = anyhow::Error;
fn from_str(s: &str) -> Result<Self> {
match s {
"noindex" => Ok(RobotsMeta::NoIndex),
"nofollow" => Ok(RobotsMeta::NoFollow),
- _ => Err(Error::UnknownRobotsMetaTag),
+ _ => Err(Error::UnknownRobotsMetaTag.into()),
}
}
}
@@ -304,12 +303,12 @@ pub struct Html {
}
impl Html {
- pub fn parse(html: &str, url: &str) -> Self {
- let mut html = Self::parse_without_text(html, url);
+ pub fn parse(html: &str, url: &str) -> Result<Self> {
+ let mut html = Self::parse_without_text(html, url)?;
html.parse_text();
- html
+ Ok(html)
}
#[cfg(test)]
@@ -317,10 +316,11 @@ impl Html {
self.clean_text = Some(text);
}
- pub fn parse_without_text(html: &str, url: &str) -> Self {
+ pub fn parse_without_text(html: &str, url: &str) -> Result<Self> {
let root = kuchiki::parse_html().one(html);
- let mut url: Url = url.to_string().into();
- url = url.full_without_id_tags().into();
+
+ let mut url = Url::parse(url)?;
+ url.set_fragment(None); // remove fragment (e.g. #comments
let mut res = Self {
root,
@@ -335,10 +335,28 @@ impl Html {
res.url = url;
}
- res.url.remove_utm_queries();
+ let queries: Vec<_> = res
+ .url
+ .query_pairs()
+ .filter(|(key, _)| !key.starts_with("utm_"))
+ .map(|(key, value)| (key.to_string(), value.to_string()))
+ .collect();
+
+ {
+ let mut query_mut = res.url.query_pairs_mut();
+ query_mut.clear();
+ if !queries.is_empty() {
+ query_mut.extend_pairs(queries);
+ }
+ }
+
+ if res.url.query().unwrap_or_default().is_empty() {
+ res.url.set_query(None);
+ }
+
res.robots = res.parse_robots_meta();
- res
+ Ok(res)
}
fn canonical_url(&self) -> Option<Url> {
@@ -349,10 +367,14 @@ impl Html {
if let Some(rel) = element.attributes.borrow().get("rel") {
if rel == "canonical" {
if let Some(href) = element.attributes.borrow().get("href") {
- let url = Url::from(href).into_absolute(self.url());
- if url.is_valid_uri() {
- canonical_url = Some(url);
- }
+ match Url::parse(href) {
+ Ok(url) => canonical_url = Some(url),
+ Err(_) => {
+ if let Ok(url) = self.url().join(href) {
+ canonical_url = Some(url);
+ }
+ }
+ };
}
}
}
@@ -449,12 +471,15 @@ impl Html {
continue;
}
- links.push(Link {
- source: self.url().clone(),
- destination: Url::from(dest.to_string())
- .into_absolute(self.url()),
- text: text.trim().to_string(),
- });
+ if let Ok(dest) =
+ Url::parse(dest).or_else(|_| self.url().join(dest))
+ {
+ links.push(Link {
+ source: self.url().clone(),
+ destination: dest,
+ text: text.trim().to_string(),
+ });
+ }
}
}
}
@@ -483,11 +508,17 @@ impl Html {
}
if let Some(dest) = attributes.borrow().get("href") {
- links.push(Link {
- source: self.url().clone(),
- destination: Url::from(dest.to_string()).into_absolute(self.url()),
- text: text.trim().to_string(),
- });
+ if dest.starts_with("mailto:") || dest.starts_with("tel:") {
+ continue;
+ }
+
+ if let Ok(dest) = Url::parse(dest).or_else(|_| self.url().join(dest)) {
+ links.push(Link {
+ source: self.url().clone(),
+ destination: dest,
+ text: text.trim().to_string(),
+ });
+ }
}
}
@@ -500,11 +531,13 @@ impl Html {
for node in self.root.select("link").unwrap() {
if let Some(element) = node.as_node().as_element() {
if let Some(href) = element.attributes.borrow().get("href") {
- links.push(Link {
- source: self.url().clone(),
- destination: Url::from(href).into_absolute(self.url()),
- text: String::new(),
- })
+ if let Ok(href) = Url::parse(href).or_else(|_| self.url().join(href)) {
+ links.push(Link {
+ source: self.url().clone(),
+ destination: href,
+ text: String::new(),
+ });
+ }
}
}
}
@@ -531,11 +564,15 @@ impl Html {
| "twitter:image:src"
) {
if let Some(content) = metadata.get("content") {
- return Some(Link {
- source: self.url().clone(),
- destination: Url::from(content.as_str()).into_absolute(self.url()),
- text: String::new(),
- });
+ if let Ok(destination) = Url::parse(content.as_str())
+ .or_else(|_| self.url().join(content.as_str()))
+ {
+ return Some(Link {
+ source: self.url().clone(),
+ destination,
+ text: String::new(),
+ });
+ }
}
}
}
@@ -551,11 +588,15 @@ impl Html {
| "vb_meta_bburl"
) {
if let Some(content) = metadata.get("content") {
- return Some(Link {
- source: self.url().clone(),
- destination: Url::from(content.as_str()).into_absolute(self.url()),
- text: String::new(),
- });
+ if let Ok(destination) = Url::parse(content.as_str())
+ .or_else(|_| self.url().join(content.as_str()))
+ {
+ return Some(Link {
+ source: self.url().clone(),
+ destination,
+ text: String::new(),
+ });
+ }
}
}
}
@@ -571,8 +612,11 @@ impl Html {
links.extend(self.scripts().into_iter().filter_map(|script| {
match script.attributes.get("src") {
Some(url) => {
- let script_url = Url::from(url.as_str()).into_absolute(self.url());
- if self.url().domain() != script_url.domain() {
+ let script_url = Url::parse(url.as_str())
+ .or_else(|_| self.url().join(url.as_str()))
+ .ok()?;
+
+ if script_url.domain() != self.url().domain() {
Some(Link {
source: self.url().clone(),
destination: script_url,
@@ -611,7 +655,7 @@ impl Html {
};
let image_type = node.attributes.borrow().get("type").map(|t| t.to_string());
- let link = Url::from(link.to_string()).into_absolute(self.url());
+ let link = Url::parse(link).or_else(|_| self.url().join(link)).ok()?;
let favicon = FaviconLink {
link,
@@ -704,7 +748,7 @@ impl Html {
let title = self.title();
if title.is_none() {
- return Err(Error::EmptyField("title"));
+ return Err(Error::EmptyField("title").into());
}
let title = title.unwrap();
@@ -715,7 +759,7 @@ impl Html {
let all_text = self.all_text();
if all_text.is_none() {
- return Err(Error::EmptyField("all body"));
+ return Err(Error::EmptyField("all body").into());
}
let all_text = all_text.unwrap();
@@ -728,17 +772,23 @@ impl Html {
}
fn pretokenize_url(&self) -> PreTokenizedString {
- let url = self.url().full();
+ let url = self.url().to_string();
self.pretokenize_string(url)
}
fn pretokenize_domain(&self) -> PreTokenizedString {
- let domain = self.url().domain().to_string();
+ let mut domain = self.url().domain().unwrap_or_default().to_string();
+
+ if let Some(stripped) = domain.strip_prefix("www.") {
+ domain = stripped.to_string();
+ }
+
self.pretokenize_string(domain)
}
fn pretokenize_site(&self) -> PreTokenizedString {
- let site = self.url().site().to_string();
+ let site = self.url().host_str().unwrap_or_default().to_string();
+
self.pretokenize_string(site)
}
@@ -797,10 +847,15 @@ impl Html {
},
};
- let site_hash = split_u128(hash(self.url().site()).0);
- let url_without_query_hash = split_u128(hash(self.url().strip_query()).0);
- let url_hash = split_u128(hash(self.url().full()).0);
- let domain_hash = split_u128(hash(self.url().domain()).0);
+ let site_hash = split_u128(hash(self.url().host_str().unwrap_or_default()).0);
+
+ let mut url_without_query = self.url().clone();
+ url_without_query.set_query(None);
+
+ let url_without_query_hash = split_u128(hash(url_without_query.as_str()).0);
+ let url_hash = split_u128(hash(self.url().as_str()).0);
+
+ let domain_hash = split_u128(hash(self.url().domain().unwrap_or_default()).0);
let title_hash = split_u128(hash(self.title().unwrap_or_default()).0);
for field in &ALL_FIELDS {
@@ -864,7 +919,7 @@ impl Html {
doc.add_pre_tokenized_text(tantivy_field, url.clone())
}
Field::Text(TextField::UrlNoTokenizer) => {
- let url = self.url().full();
+ let url = self.url().to_string();
doc.add_pre_tokenized_text(
tantivy_field,
@@ -880,7 +935,7 @@ impl Html {
},
);
}
- Field::Text(TextField::Site) => {
+ Field::Text(TextField::SiteWithout) => {
doc.add_pre_tokenized_text(tantivy_field, site.clone())
}
Field::Text(TextField::Domain) => {
@@ -889,27 +944,27 @@ impl Html {
Field::Text(TextField::SiteNoTokenizer) => doc.add_pre_tokenized_text(
tantivy_field,
PreTokenizedString {
- text: self.url().site().to_string(),
+ text: site.text.clone(),
tokens: vec![tantivy::tokenizer::Token {
offset_from: 0,
- offset_to: self.url().site().len(),
+ offset_to: site.text.len(),
position: 0,
- text: self.url().site().to_string(),
+ text: site.text.clone(),
position_length: 1,
}],
},
),
Field::Text(TextField::SiteIfHomepageNoTokenizer) => {
- if self.url().is_homepage() {
+ if self.is_homepage() {
doc.add_pre_tokenized_text(
tantivy_field,
PreTokenizedString {
- text: self.url().site().to_string(),
+ text: site.text.clone(),
tokens: vec![tantivy::tokenizer::Token {
offset_from: 0,
- offset_to: self.url().site().len(),
+ offset_to: site.text.len(),
position: 0,
- text: self.url().site().to_string(),
+ text: site.text.clone(),
position_length: 1,
}],
},
@@ -921,41 +976,49 @@ impl Html {
Field::Text(TextField::DomainNoTokenizer) => doc.add_pre_tokenized_text(
tantivy_field,
PreTokenizedString {
- text: self.url().domain().to_string(),
+ text: domain.text.clone(),
tokens: vec![tantivy::tokenizer::Token {
offset_from: 0,
- offset_to: self.url().domain().len(),
+ offset_to: domain.text.len(),
position: 0,
- text: self.url().domain().to_string(),
+ text: domain.text.clone(),
position_length: 1,
}],
},
),
Field::Text(TextField::TitleIfHomepage) => {
- if self.url().is_homepage() {
+ if self.is_homepage() {
doc.add_pre_tokenized_text(tantivy_field, title.clone());
} else {
doc.add_text(tantivy_field, "");
}
}
Field::Text(TextField::DomainIfHomepage) => {
- if self.url().is_homepage() {
- doc.add_text(tantivy_field, self.url().domain());
+ if self.is_homepage() {
+ doc.add_text(tantivy_field, domain.text.clone());
} else {
doc.add_text(tantivy_field, "");
}
}
Field::Text(TextField::DomainNameIfHomepageNoTokenizer) => {
- if self.url().is_homepage() {
+ if self.is_homepage() {
+ let domain_name = self
+ .url()
+ .domain()
+ .unwrap_or_default()
+ .find('.')
+ .map(|index| &domain.text[..index])
+ .unwrap_or_default();
+
doc.add_pre_tokenized_text(
tantivy_field,
PreTokenizedString {
- text: self.url().domain_name().to_string(),
+ text: domain_name.to_string(),
tokens: vec![tantivy::tokenizer::Token {
offset_from: 0,
- offset_to: self.url().domain_name().len(),
+ offset_to: domain_name.len(),
position: 0,
- text: self.url().domain_name().to_string(),
+ text: domain_name.to_string(),
position_length: 1,
}],
},
@@ -974,7 +1037,7 @@ impl Html {
doc.add_pre_tokenized_text(tantivy_field, pretokenized_schema_json.clone());
}
Field::Fast(FastField::IsHomepage) => {
- doc.add_u64(tantivy_field, self.url().is_homepage().into());
+ doc.add_u64(tantivy_field, (self.is_homepage()).into());
}
Field::Fast(FastField::LastUpdated) => doc.add_u64(
tantivy_field,
@@ -1046,19 +1109,27 @@ impl Html {
Field::Fast(FastField::NumPathAndQuerySlashes) => {
let num_slashes = self
.url()
- .path_and_query()
- .chars()
- .filter(|c| *c == '/')
- .count();
+ .path_segments()
+ .map(|segments| segments.count())
+ .unwrap_or(0);
+
doc.add_u64(tantivy_field, num_slashes as u64);
}
Field::Fast(FastField::NumPathAndQueryDigits) => {
let num_digits = self
.url()
- .path_and_query()
+ .path()
.chars()
.filter(|c| c.is_ascii_digit())
- .count();
+ .count()
+ + self
+ .url()
+ .query()
+ .unwrap_or_default()
+ .chars()
+ .filter(|c| c.is_ascii_digit())
+ .count();
+
doc.add_u64(tantivy_field, num_digits as u64);
}
Field::Text(TextField::BacklinkText)
@@ -1106,26 +1177,39 @@ impl Html {
let mut links: Vec<Url> = Vec::new();
for script in self.scripts() {
- if let Some(link) = script.attributes.get("src") {
- links.push(Url::from(link.to_string()).into_absolute(self.url()));
+ if let Some(link) = script
+ .attributes
+ .get("src")
+ .and_then(|link| Url::parse(link).or_else(|_| self.url().join(link)).ok())
+ {
+ links.push(link);
}
for res in URL_REGEX.find_iter(&script.content) {
- links.push(Url::from(res.as_str().to_string()).into_absolute(self.url()));
+ if let Ok(link) =
+ Url::parse(res.as_str()).or_else(|_| self.url().join(res.as_str()))
+ {
+ links.push(link);
+ }
}
}
for node in self.root.select("link").unwrap() {
- if let Some(link) = node.attributes.borrow().get("href") {
- links.push(Url::from(link.to_string()).into_absolute(self.url()));
+ if let Some(link) = node
+ .attributes
+ .borrow()
+ .get("href")
+ .and_then(|link| Url::parse(link).or_else(|_| self.url().join(link)).ok())
+ {
+ links.push(link);
}
}
links
.into_iter()
- .filter(|link| !link.site().is_empty())
- .filter(|link| link.site() != self.url().site())
- .unique_by(|link| link.site().to_string())
+ .filter(|link| link.host_str().is_some())
+ .filter(|link| link.host_str() != self.url().host_str())
+ .unique_by(|link| link.host_str().unwrap().to_string())
.collect()
}
@@ -1173,7 +1257,11 @@ impl Html {
false
}
})
- .and_then(|metadata| metadata.get("content").map(|link| link.clone().into()))
+ .and_then(|metadata| {
+ metadata
+ .get("content")
+ .and_then(|link| Url::parse(link).or_else(|_| self.url().join(link)).ok())
+ })
.map(|url| ImageLink {
url,
title: self.og_title(),
@@ -1193,7 +1281,7 @@ impl Html {
.many()
.into_iter()
.filter_map(|url| url.try_into_string())
- .map(|url| url.into())
+ .filter_map(|url| Url::parse(&url).or_else(|_| self.url().join(&url)).ok())
})
})
.flatten()
@@ -1218,22 +1306,16 @@ impl Html {
}
pub fn primary_image(&self) -> Option<ImageLink> {
- self.og_image()
- .or_else(|| {
- self.schema_org_images()
- .first()
- .cloned()
- .map(|url| ImageLink {
- url,
- title: self.og_title(),
- description: self.description(),
- })
- })
- .map(|mut image| {
- image.url = image.url.into_absolute(self.url());
-
- image
- })
+ self.og_image().or_else(|| {
+ self.schema_org_images()
+ .first()
+ .cloned()
+ .map(|url| ImageLink {
+ url,
+ title: self.og_title(),
+ description: self.description(),
+ })
+ })
}
pub fn og_description(&self) -> Option<String> {
@@ -1279,6 +1361,10 @@ impl Html {
})
.and_then(|metadata| metadata.get("content").cloned())
}
+
+ pub fn is_homepage(&self) -> bool {
+ self.url().path() == "/"
+ }
}
fn stemmer_from_lang(lang: &Lang) -> rust_stemmers::Stemmer {
@@ -1321,12 +1407,6 @@ pub struct Link {
pub text: String,
}
-impl Link {
- pub fn matches_url_regex(&self) -> bool {
- self.destination.matches_url_regex() && self.source.matches_url_regex()
- }
-}
-
pub type Meta = HashMap<String, String>;
#[cfg(test)]
@@ -1357,15 +1437,15 @@ mod tests {
"#
);
- let webpage = Html::parse(&raw, "https://www.example.com/whatever");
+ let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();
assert_eq!(webpage.title(), Some("Best website".to_string()));
assert_eq!(
webpage.anchor_links(),
vec![Link {
- source: "https://www.example.com/whatever".to_string().into(),
- destination: "https://example.com".to_string().into(),
+ source: Url::parse("https://www.example.com/whatever").unwrap(),
+ destination: Url::parse("https://example.com").unwrap(),
text: "Link to example".to_string()
}]
);
@@ -1380,8 +1460,8 @@ mod tests {
webpage.url().to_string().as_str(),
"https://www.example.com/whatever"
);
- assert_eq!(webpage.url().site(), "www.example.com");
- assert_eq!(webpage.url().domain(), "example.com");
+ assert_eq!(webpage.url().host_str().unwrap(), "www.example.com");
+ assert_eq!(webpage.url().domain().unwrap(), "www.example.com");
}
#[test]
@@ -1399,7 +1479,7 @@ mod tests {
"#
);
- let webpage = Html::parse(&raw, "https://www.example.com/whatever");
+ let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();
assert_eq!(webpage.title(), None);
}
@@ -1416,7 +1496,7 @@ mod tests {
"#
);
- let webpage = Html::parse(&raw, "https://www.example.com/whatever");
+ let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();
assert_eq!(webpage.clean_text(), Some(&CONTENT.to_string()));
}
@@ -1444,7 +1524,7 @@ mod tests {
"#
);
- let webpage = Html::parse(&raw, "https://www.example.com");
+ let webpage = Html::parse(&raw, "https://www.example.com").unwrap();
assert!(!webpage.clean_text().unwrap().contains("not"));
}
@@ -1472,7 +1552,7 @@ mod tests {
"#
);
- let webpage = Html::parse(&raw, "https://www.example.com");
+ let webpage = Html::parse(&raw, "https://www.example.com").unwrap();
assert!(!webpage.clean_text().unwrap().contains("not"));
}
@@ -1481,37 +1561,44 @@ mod tests {
fn co_uk_domain() {
let raw = "";
- let webpage = Html::parse(raw, "https://www.domain.co.uk");
- assert_eq!(webpage.url().domain(), "domain.co.uk");
+ let webpage = Html::parse(raw, "https://www.domain.co.uk").unwrap();
+ assert_eq!(
+ webpage.url().domain().unwrap_or_default(),
+ "www.domain.co.uk"
+ );
}
#[test]
fn is_homepage() {
- let webpage = Html::parse("", "https://www.example.com");
- assert!(webpage.url().is_homepage());
+ let webpage = Html::parse("", "https://www.example.com").unwrap();
+ assert!(webpage.is_homepage());
- let webpage = Html::parse("", "https://www.example.com/");
- assert!(webpage.url().is_homepage());
+ let webpage = Html::parse("", "https://www.example.com/").unwrap();
+ assert!(webpage.is_homepage());
- let webpage = Html::parse("", "https://www.example.com/test");
- assert!(!webpage.url().is_homepage());
+ let webpage = Html::parse("", "https://www.example.com/test").unwrap();
+ assert!(!webpage.is_homepage());
- let webpage = Html::parse("", "https://example.com/test");
- assert!(!webpage.url().is_homepage());
+ let webpage = Html::parse("", "https://example.com/test").unwrap();
+ assert!(!webpage.is_homepage());
- let webpage = Html::parse("", "https://example.com/");
- assert!(webpage.url().is_homepage());
+ let webpage = Html::parse("", "https://example.com/").unwrap();
+ assert!(webpage.is_homepage());
- let webpage = Html::parse("", "https://example.com");
- assert!(webpage.url().is_homepage());
+ let webpage = Html::parse("", "https://example.com").unwrap();
+ assert!(webpage.is_homepage());
- let webpage = Html::parse("", "http://example.com");
- assert!(webpage.url().is_homepage());
+ let webpage = Html::parse("", "http://example.com").unwrap();
+ assert!(webpage.is_homepage());
}
#[test]
fn hard_parsing() {
- let webpage = Html::parse(include_str!("../../testcases/parsing/yasudaya.html"), "");
+ let webpage = Html::parse(
+ include_str!("../../testcases/parsing/yasudaya.html"),
+ "https://example.com",
+ )
+ .unwrap();
assert_eq!(
webpage.title(),
Some("パチンコ大当たり情報 - Pジューシーハニー3 大当たり詳細ページ - やすだひばりヶ丘店".to_string())
@@ -1519,7 +1606,11 @@ mod tests {
assert!(webpage.all_text().is_some());
assert!(!webpage.all_text().unwrap().is_empty());
- let webpage = Html::parse(include_str!("../../testcases/parsing/5390001.html"), "");
+ let webpage = Html::parse(
+ include_str!("../../testcases/parsing/5390001.html"),
+ "https://example.com",
+ )
+ .unwrap();
assert_eq!(
webpage.title(),
Some("特效烟机系列_山东壹线文化传播有限公司".to_string())
@@ -1529,8 +1620,9 @@ mod tests {
let webpage = Html::parse(
include_str!("../../testcases/parsing/77p2p-7.live-105.html"),
- "",
- );
+ "https://example.com",
+ )
+ .unwrap();
assert_eq!(
webpage.title(),
Some("77p2pЅu¤WЖ[¬Э - ҐDјЅ :: іnєс".to_string())
@@ -1541,7 +1633,11 @@ mod tests {
#[test]
fn reddit_comments() {
- let webpage = Html::parse(include_str!("../../testcases/parsing/reddit.html"), "");
+ let webpage = Html::parse(
+ include_str!("../../testcases/parsing/reddit.html"),
+ "https://reddit.com/",
+ )
+ .unwrap();
assert!(webpage.clean_text().is_some());
assert!(webpage.clean_text().unwrap().len() > 1000);
@@ -1555,8 +1651,9 @@ mod tests {
fn out_of_bounds_str() {
let webpage = Html::parse(
include_str!("../../testcases/parsing/byte_index_out_of_bounds.html"),
- "",
- );
+ "https://example.com",
+ )
+ .unwrap();
assert_eq!(webpage.title(), Some("Test".to_string()));
assert!(webpage.all_text().is_some());
assert!(!webpage.all_text().unwrap().is_empty());
@@ -1565,18 +1662,6 @@ mod tests {
webpage.into_tantivy(&schema).unwrap();
}
- #[test]
- fn test_find_protocol() {
- assert_eq!(
- Url::from("https://example.com".to_string()).protocol(),
- "https"
- );
- assert_eq!(
- Url::from("http://example.com".to_string()).protocol(),
- "http"
- );
- }
-
#[test]
fn simple_favicon() {
let raw = r#"
@@ -1588,11 +1673,11 @@ mod tests {
"#
.to_string();
- let webpage = Html::parse(&raw, "https://www.example.com");
+ let webpage = Html::parse(&raw, "https://www.example.com").unwrap();
assert_eq!(
webpage.favicon(),
Some(FaviconLink {
- link: "https://example.com/favicon.png".to_string().into(),
+ link: Url::parse("https://example.com/favicon.png").unwrap(),
width: Some(192),
height: Some(192),
image_type: None
@@ -1611,11 +1696,11 @@ mod tests {
"#
);
- let webpage = Html::parse(&raw, site_url);
+ let webpage = Html::parse(&raw, site_url).unwrap();
assert_eq!(
webpage.favicon(),
Some(FaviconLink {
- link: expected.to_string().into(),
+ link: Url::parse(expected).unwrap(),
width: Some(192),
height: Some(192),
image_type: None
@@ -1647,7 +1732,7 @@ mod tests {
);
full_link_favicon(
"favicon.png",
- "https://www.example.com/test",
+ "https://www.example.com/test/",
"https://www.example.com/test/favicon.png",
);
full_link_favicon(
@@ -1659,8 +1744,8 @@ mod tests {
#[test]
fn domain_from_domain_url() {
- let url: Url = "example.com".to_string().into();
- assert_eq!(url.domain(), "example.com");
+ let url: Url = Url::parse("http://example.com").unwrap();
+ assert_eq!(url.domain().unwrap(), "example.com");
}
#[test]
@@ -1674,7 +1759,7 @@ mod tests {
</body>
</html>
"#;
- let html = Html::parse(html, "example.com");
+ let html = Html::parse(html, "https://example.com").unwrap();
assert_eq!(
html.updated_time(),
@@ -1689,7 +1774,7 @@ mod tests {
</body>
</html>
"#;
- let html = Html::parse(html, "example.com");
+ let html = Html::parse(html, "https://example.com").unwrap();
assert_eq!(html.updated_time(), None);
@@ -1702,7 +1787,7 @@ mod tests {
</body>
</html>
"#;
- let html = Html::parse(html, "example.com");
+ let html = Html::parse(html, "https://example.com").unwrap();
assert_eq!(html.updated_time(), None);
@@ -1715,7 +1800,7 @@ mod tests {
</body>
</html>
"#;
- let html = Html::parse(html, "example.com");
+ let html = Html::parse(html, "https://example.com").unwrap();
assert_eq!(html.updated_time(), None);
}
@@ -1733,12 +1818,12 @@ mod tests {
</body>
</html>
"#;
- let html = Html::parse(html, "example.com");
+ let html = Html::parse(html, "https://example.com").unwrap();
assert_eq!(
html.primary_image(),
Some(ImageLink {
- url: "https://example.com/link_to_image.html".to_string().into(),
+ url: Url::parse("https://example.com/link_to_image.html").unwrap(),
title: Some("title".to_string()),
description: Some("desc".to_string())
})
@@ -1752,7 +1837,7 @@ mod tests {
</body>
</html>
"#;
- let html = Html::parse(html, "example.com");
+ let html = Html::parse(html, "https://example.com").unwrap();
assert_eq!(html.primary_image(), None);
@@ -1776,12 +1861,12 @@ mod tests {
</body>
</html>
"#;
- let html = Html::parse(html, "https://example.com");
+ let html = Html::parse(html, "https://example.com").unwrap();
assert_eq!(
html.primary_image(),
Some(ImageLink {
- url: "https://example.com/mexico-beach.jpg".to_string().into(),
+ url: Url::parse("https://example.com/mexico-beach.jpg").unwrap(),
title: None,
description: None
})
@@ -1799,7 +1884,7 @@ mod tests {
</body>
</html>
"#;
- let html = Html::parse(html, "example.com");
+ let html = Html::parse(html, "http://example.com").unwrap();
assert_eq!(html.updated_time(), None);
}
@@ -1815,7 +1900,7 @@ mod tests {
</body>
</html>
"#;
- let html = Html::parse(html, "example.com");
+ let html = Html::parse(html, "http://example.com").unwrap();
assert_eq!(
html.description(),
@@ -1830,7 +1915,7 @@ mod tests {
</body>
</html>
"#;
- let html = Html::parse(html, "example.com");
+ let html = Html::parse(html, "http://example.com").unwrap();
assert_eq!(html.description(), None);
}
@@ -1846,7 +1931,7 @@ mod tests {
</body>
</html>
"#;
- let html = Html::parse(html, "example.com");
+ let html = Html::parse(html, "http://example.com").unwrap();
assert_eq!(
html.updated_time(),
@@ -1881,12 +1966,12 @@ mod tests {
</body>
</html>
"#;
- let html = Html::parse(html, "example.com");
+ let html = Html::parse(html, "http://example.com").unwrap();
assert_eq!(
html.trackers()
.into_iter()
- .map(|url| url.site().to_string())
+ .map(|url| url.host_str().unwrap().to_string())
.collect::<Vec<_>>(),
vec![
"cdn.segment.com".to_string(),
@@ -1924,8 +2009,8 @@ mod tests {
</body>
</html>
"#,
- "example.com",
- );
+ "https://example.com",
+ ).unwrap();
assert_eq!(html.title(), Some("Test site".to_string()));
assert_eq!(html.all_text(), Some("test".to_string()));
@@ -1944,8 +2029,9 @@ mod tests {
</body>
</html>
"#,
- "example.com",
- );
+ "https://example.com",
+ )
+ .unwrap();
let webpage = Webpage {
html,
@@ -1978,8 +2064,9 @@ mod tests {
</body>
</html>
"#,
- "example.com",
- );
+ "http://example.com",
+ )
+ .unwrap();
let webpage = Webpage {
html,
backlinks: Vec::new(),
@@ -2013,15 +2100,15 @@ mod tests {
"#
);
- let webpage = Html::parse(&raw, "https://www.example.com/whatever");
+ let webpage = Html::parse(&raw, "https://www.example.com/whatever").unwrap();
assert_eq!(webpage.title(), Some("Best website".to_string()));
assert_eq!(
webpage.anchor_links(),
vec![Link {
- source: "https://www.example.com/whatever".to_string().into(),
- destination: "https://example.com".to_string().into(),
+ source: Url::parse("https://www.example.com/whatever").unwrap(),
+ destination: Url::parse("https://example.com").unwrap(),
text: "Link to example".to_string()
},]
);
@@ -2030,7 +2117,7 @@ mod tests {
#[test]
fn stackoverflow_question_has_clean_text() {
let stackoverflow = include_str!("../../testcases/schema_org/stackoverflow_with_code.html");
- let html = Html::parse(stackoverflow, "https://www.example.com");
+ let html = Html::parse(stackoverflow, "https://www.example.com").unwrap();
assert!(html.clean_text().is_some());
}
@@ -2048,16 +2135,17 @@ mod tests {
</html>
"#,
"https://www.example.com/whatever",
- );
+ )
+ .unwrap();
assert_eq!(
html.canonical_url(),
- Some("https://example.com/canonical.html".to_string().into())
+ Some(Url::parse("https://example.com/canonical.html").unwrap())
);
assert_eq!(
html.url(),
- &"https://example.com/canonical.html".to_string().into()
+ &Url::parse("https://example.com/canonical.html").unwrap()
);
let html = Html::parse(
@@ -2070,12 +2158,13 @@ mod tests {
</html>
"#,
"https://www.example.com/whatever",
- );
+ )
+ .unwrap();
assert_eq!(html.canonical_url(), None);
assert_eq!(
html.url(),
- &"https://www.example.com/whatever".to_string().into()
+ &Url::parse("https://www.example.com/whatever").unwrap()
);
}
@@ -2092,7 +2181,8 @@ mod tests {
</html>
"#,
"https://www.example.com/whatever",
- );
+ )
+ .unwrap();
assert!(html.is_no_index());
assert!(html.is_no_follow());
@@ -2108,7 +2198,8 @@ mod tests {
</html>
"#,
"https://www.example.com/whatever",
- );
+ )
+ .unwrap();
assert!(html.is_no_index());
assert!(html.is_no_follow());
@@ -2124,7 +2215,8 @@ mod tests {
</html>
"#,
"https://www.example.com/whatever",
- );
+ )
+ .unwrap();
assert!(html.is_no_index());
assert!(!html.is_no_follow());
@@ -2140,7 +2232,8 @@ mod tests {
</html>
"#,
"https://www.example.com/whatever",
- );
+ )
+ .unwrap();
assert!(!html.is_no_index());
assert!(html.is_no_follow());
@@ -2155,7 +2248,8 @@ mod tests {
</html>
"#,
"https://www.example.com/whatever",
- );
+ )
+ .unwrap();
assert!(!html.is_no_index());
assert!(!html.is_no_follow());
diff --git a/core/src/webpage/region.rs b/core/src/webpage/region.rs
index 8436f8df..ecfbf648 100644
--- a/core/src/webpage/region.rs
+++ b/core/src/webpage/region.rs
@@ -88,7 +88,7 @@ impl Region {
"ger" => Ok(Region::Germany),
"spa" => Ok(Region::Spain),
"us" => Ok(Region::US),
- _ => Err(Error::UnknownRegion),
+ _ => Err(Error::UnknownRegion.into()),
}
}
@@ -100,9 +100,9 @@ impl Region {
whatlang::Lang::Fra => Ok(Region::France),
whatlang::Lang::Deu => Ok(Region::Germany),
whatlang::Lang::Dan => Ok(Region::Denmark),
- _ => Err(Error::UnknownRegion),
+ _ => Err(Error::UnknownRegion.into()),
},
- None => Err(Error::UnknownRegion),
+ None => Err(Error::UnknownRegion.into()),
}
}
diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rs
deleted file mode 100644
index e08fb475..00000000
--- a/core/src/webpage/url.rs
+++ /dev/null
@@ -1,620 +0,0 @@
-// Stract is an open source web search engine.
-// Copyright (C) 2023 Stract ApS
-//
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as
-// published by the Free Software Foundation, either version 3 of the
-// License, or (at your option) any later version.
-//
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-// GNU Affero General Public License for more details.
-//
-// You should have received a copy of the GNU Affero General Public License
-// along with this program. If not, see <https://www.gnu.org/licenses/>.
-
-use std::{fmt::Display, ops::Range, time::Duration};
-
-use publicsuffix::Psl;
-use serde::{Deserialize, Serialize};
-use tracing::debug;
-
-use crate::{ceil_char_boundary, floor_char_boundary};
-
-use super::URL_REGEX;
-
-pub static LIST: once_cell::sync::Lazy<publicsuffix::List> = once_cell::sync::Lazy::new(|| {
- include_str!("../../public_suffix_list.dat")
- .parse()
- .expect("Failed to parse public suffix list")
-});
-
-#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Clone, Hash)]
-pub struct Url(String);
-
-impl Display for Url {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
- write!(f, "{}", &self.0)
- }
-}
-
-impl From<String> for Url {
- fn from(url: String) -> Self {
- let url = if url.ends_with('/') {
- &url[..url.len() - 1]
- } else {
- &url
- };
-
- Url(url.trim().to_string())
- }
-}
-
-impl From<&str> for Url {
- fn from(url: &str) -> Self {
- Self::from(url.to_string())
- }
-}
-
-impl From<Url> for String {
- fn from(value: Url) -> Self {
- value.0
- }
-}
-
-impl Url {
- pub fn normalize(&self) -> Self {
- let mut res: Url = self.strip_protocol().to_lowercase().into();
-
- if let Some(stripped) = res.0.strip_prefix("www.") {
- res = Url(stripped.to_string())
- }
-
- res.remove_utm_queries();
-
- res
- }
-
- pub fn remove_utm_queries(&mut self) {
- let mut new_url = None;
-
- if let Some(queries) = self.queries() {
- let mut tmp_new_url = self.strip_query().to_string();
- let mut new_queries = Vec::new();
- for (key, value) in queries {
- if !key.starts_with("utm_") {
- new_queries.push((key, value));
- }
- }
-
- if !new_queries.is_empty() {
- tmp_new_url.push('?');
- for (key, value) in new_queries {
- tmp_new_url.push_str(key);
-
- if let Some(value) = value {
- tmp_new_url.push('=');
- tmp_new_url.push_str(value);
- }
-
- tmp_new_url.push('&');
- }
- tmp_new_url.pop();
- }
-
- new_url = Some(tmp_new_url);
- }
-
- if let Some(new_url) = new_url {
- self.0 = new_url;
- }
- }
-
- pub fn queries(&self) -> Option<impl Iterator<Item = (&str, Option<&str>)> + '_> {
- match self.query_range() {
- Some(range) => {
- // remove the '?'
- let query = &self.0[ceil_char_boundary(&self.0, range.start + 1)
- ..ceil_char_boundary(&self.0, range.end)];
-
- Some(query.split('&').filter_map(|query| {
- let mut split = query.split('=');
- let key = split.next()?;
- let value = split.next();
- Some((key, value))
- }))
- }
- None => None,
- }
- }
-
- pub fn strip_protocol(&self) -> &str {
- let mut start_host = 0;
- let url = &self.0;
-
- if url.starts_with("http://") {
- start_host = 7;
- } else if url.starts_with("https://") {
- start_host = 8;
- } else if url.starts_with("//") {
- start_host = 2;
- }
-
- let start_host = floor_char_boundary(url, std::cmp::min(start_host, url.len()));
- &url[start_host..]
- }
-
- fn query_range(&self) -> Option<Range<usize>> {
- let url = &self.0;
-
- match url.find('?') {
- Some(start_query) => {
- let start_query = ceil_char_boundary(url, start_query);
- let mut end_query = url.len();
-
- if let Some(hash) = url.find('#') {
- if hash > start_query {
- end_query = ceil_char_boundary(url, hash);
- }
- }
-
- Some(start_query..end_query)
- }
- None => None,
- }
- }
-
- pub fn strip_query(&self) -> &str {
- let url = &self.0;
-
- if let Some(query_range) = self.query_range() {
- &url[..query_range.start]
- } else {
- url
- }
- }
-
- pub fn site(&self) -> &str {
- let url = self.strip_protocol();
-
- let mut end_site = url.len();
- if url.contains('/') {
- end_site = url.find('/').expect("The url contains atleast 1 '/'");
- }
-
- let end_site = ceil_char_boundary(url, end_site);
- let res = &url[..end_site];
-
- if let Some(query_begin) = res.find('?') {
- let query_begin = ceil_char_boundary(res, query_begin);
- &res[..query_begin]
- } else {
- res
- }
- }
- pub fn domain(&self) -> &str {
- let site = self.site().as_bytes();
- match LIST.domain(site) {
- Some(domain) => match std::str::from_utf8(domain.as_bytes()) {
- Ok(res) => res,
- Err(_) => "",
- },
- None => "",
- }
- }
-
- fn tld(&self) -> &str {
- let site = self.site().as_bytes();
- match LIST.suffix(site) {
- Some(tld) => match std::str::from_utf8(tld.as_bytes()) {
- Ok(res) => res,
- Err(_) => "",
- },
- None => "",
- }
- }
-
- pub fn domain_name(&self) -> &str {
- let domain = self.domain();
- let tld = self.tld();
-
- if domain.is_empty() || tld.is_empty() {
- ""
- } else {
- if tld.len() + 1 > domain.len() {
- return "";
- }
-
- let idx = domain.len() - tld.len() - 1;
- let idx = ceil_char_boundary(domain, idx);
- &domain[..idx]
- }
- }
-
- pub fn subdomain(&self) -> Option<&str> {
- if let Some(subdomain) = self.site().strip_suffix(self.domain()) {
- if subdomain.is_empty() || subdomain == "." {
- None
- } else {
- let idx = floor_char_boundary(subdomain, subdomain.len() - 1);
- Some(&subdomain[..idx])
- }
- } else {
- None
- }
- }
-
- pub fn is_homepage(&self) -> bool {
- let url = self.strip_protocol();
- match url.find('/') {
- Some(idx) => idx == url.len() - 1,
- None => true,
- }
- }
-
- fn find_protocol_end(&self) -> usize {
- let mut start_host = 0;
- let url = &self.0;
- if url.starts_with("http://") || url.starts_with("https://") {
- start_host = url
- .find(':')
- .expect("It was checked that url starts with protocol");
- } else if url.starts_with("//") {
- start_host = url
- .find('/')
- .expect("It was checked that url starts with protocol")
- + 1;
- }
- start_host
- }
- pub fn protocol(&self) -> &str {
- &self.0[..self.find_protocol_end()]
- }
-
- pub fn is_full_path(&self) -> bool {
- matches!(self.protocol(), "http" | "https" | "pdf") || self.0.starts_with("//")
- }
-
- fn prefix_with(&mut self, url: &Url) {
- self.0 = match (url.0.ends_with('/'), self.0.starts_with('/')) {
- (true, true) => {
- let prot = url.protocol().to_string();
- if prot.is_empty() {
- url.full() + self.0.strip_prefix('/').unwrap()
- } else {
- prot + "://" + url.site() + self.0.as_str()
- }
- }
- (true, false) => url.full() + self.0.as_str(),
- (false, true) => {
- let prot = url.protocol().to_string();
- if prot.is_empty() {
- url.full() + self.0.as_str()
- } else {
- prot + "://" + url.site() + self.0.as_str()
- }
- }
- (false, false) => url.full() + "/" + self.0.as_str(),
- };
- }
-
- pub fn full(&self) -> String {
- if self.0.starts_with("//") {
- return "http:".to_string() + self.0.as_str();
- }
-
- if self.find_protocol_end() == 0 {
- "http://".to_string() + self.0.as_str()
- } else {
- self.0.clone()
- }
- }
-
- pub fn into_absolute(self, base: &Url) -> Self {
- let mut url = self;
- if !url.is_full_path() {
- url.prefix_with(base);
- }
- url
- }
-
- pub async fn download_bytes(&self, timeout: Duration) -> Option<Vec<u8>> {
- let client = reqwest::Client::builder().timeout(timeout).build().unwrap();
-
- debug!("downloading {:?}", self.full());
-
- match client.get(self.full()).send().await {
- Ok(res) => {
- let bytes = res.bytes().await.ok()?.to_vec();
- Some(bytes)
- }
- Err(_) => None,
- }
- }
-
- pub fn raw(&self) -> &str {
- &self.0
- }
-
- pub fn is_valid_uri(&self) -> bool {
- self.full().as_str().parse::<http::Uri>().is_ok()
- }
-
- pub(crate) fn host_without_www(&self) -> &str {
- if let Some(subdomain) = self.subdomain() {
- if subdomain == "www" {
- self.domain()
- } else {
- self.site()
- }
- } else {
- self.domain()
- }
- }
-
- pub fn full_without_id_tags(&self) -> String {
- let full = self.full();
- if let Some(last_slash) = full.rfind('/') {
- let last_slash = floor_char_boundary(&full, last_slash);
- if let Some(id_begin) = full[last_slash..].find('#') {
- let idx = floor_char_boundary(&full, id_begin + last_slash);
- full[..idx].to_string()
- } else {
- full
- }
- } else if let Some(id_begin) = full.find('#') {
- let idx = floor_char_boundary(&full, id_begin);
- full[..idx].to_string()
- } else {
- full
- }
- }
-
- pub fn matches_url_regex(&self) -> bool {
- URL_REGEX.is_match(&self.full())
- }
-
- pub fn without_protocol(&self) -> &str {
- self.0[self.find_protocol_end()..]
- .strip_prefix("://")
- .unwrap_or(&self.0)
- }
-
- pub fn path_ends_with(&self, ending: &str) -> bool {
- if self.is_homepage() {
- false
- } else {
- self.strip_query().ends_with(ending)
- }
- }
-
- pub fn is_empty(&self) -> bool {
- self.0.is_empty()
- }
-
- pub fn path_and_query(&self) -> &str {
- let without_prot = self.without_protocol();
- without_prot
- .find('/')
- .map_or("", |idx| without_prot.get(idx..).map_or("", |path| path))
- }
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn double_slash_start() {
- let url: Url = "//scripts.dailymail.co.uk".to_string().into();
-
- assert_eq!(url.domain(), "dailymail.co.uk");
- assert_eq!(url.domain_name(), "dailymail");
- assert_eq!(url.site(), "scripts.dailymail.co.uk");
- assert_eq!(url.full(), "http://scripts.dailymail.co.uk");
- }
-
- #[test]
- fn co_uk_edgecase() {
- let url: Url = "dailymail.co.uk".to_string().into();
-
- assert_eq!(url.domain(), "dailymail.co.uk");
- assert_eq!(url.site(), "dailymail.co.uk");
- assert_eq!(url.full().as_str(), "http://dailymail.co.uk");
- assert!(url.matches_url_regex());
- }
-
- #[test]
- fn full() {
- let url: Url = "https://example.com".to_string().into();
- assert_eq!(url.full().as_str(), "https://example.com");
- assert!(url.matches_url_regex());
-
- let url: Url = "http://example.com".to_string().into();
- assert_eq!(url.full().as_str(), "http://example.com");
- }
-
- #[test]
- fn into_absolute() {
- let mut a: Url = "/test".to_string().into();
- let b: Url = "https://example.com".to_string().into();
- a = a.into_absolute(&b);
- assert_eq!(a.full().as_str(), "https://example.com/test");
-
- let mut a: Url = "test".to_string().into();
- let b: Url = "https://example.com".to_string().into();
- a = a.into_absolute(&b);
- assert_eq!(a.full().as_str(), "https://example.com/test");
-
- let mut a: Url = "test".to_string().into();
- let b: Url = "https://example.com/".to_string().into();
- a = a.into_absolute(&b);
- assert_eq!(a.full().as_str(), "https://example.com/test");
-
- let mut a: Url = "/test".to_string().into();
- let b: Url = "https://example.com/".to_string().into();
- a = a.into_absolute(&b);
- assert_eq!(a.full().as_str(), "https://example.com/test");
-
- let mut a: Url = "https://example.com/test".to_string().into();
- let b: Url = "https://example.com/".to_string().into();
- a = a.into_absolute(&b);
- assert_eq!(a.full().as_str(), "https://example.com/test");
-
- let mut a: Url = "https://example.com/test".to_string().into();
- let b: Url = "example.com".to_string().into();
- a = a.into_absolute(&b);
- assert_eq!(a.full().as_str(), "https://example.com/test");
-
- let mut a: Url = "https://a.com/test".to_string().into();
- let b: Url = "b.com".to_string().into();
- a = a.into_absolute(&b);
- assert_eq!(a.full().as_str(), "https://a.com/test");
- }
-
- #[test]
- fn is_full_path() {
- let url: Url = "https://dailymail.co.uk".to_string().into();
- assert!(url.is_full_path());
-
- let url: Url = "//dailymail.co.uk".to_string().into();
- assert!(url.is_full_path());
- }
-
- #[test]
- fn is_valid() {
- let url: Url = "https://dailymail.co.uk".to_string().into();
- assert!(url.is_valid_uri());
-
- let url: Url = "da<>ilymail.co.uk".to_string().into();
- assert!(!url.is_valid_uri());
- }
-
- #[test]
- fn subdomain() {
- let url: Url = "https://test.example.com".to_string().into();
- assert_eq!(url.subdomain(), Some("test"));
-
- let url: Url = "https://test1.test2.example.com".to_string().into();
- assert_eq!(url.subdomain(), Some("test1.test2"));
-
- let url: Url = "https://example.com".to_string().into();
- assert_eq!(url.subdomain(), None);
- }
-
- #[test]
- fn url_without_query() {
- let url: Url = "https://test.example.com?key=val&key2=val2"
- .to_string()
- .into();
-
- assert_eq!(url.strip_query(), "https://test.example.com");
- }
-
- #[test]
- fn url_without_id() {
- let url: Url = "https://test.example.com#id".to_string().into();
- assert_eq!(&url.full_without_id_tags(), "https://test.example.com");
-
- let url: Url = "https://test.example.com/#/test".to_string().into();
- assert_eq!(
- &url.full_without_id_tags(),
- "https://test.example.com/#/test"
- );
- }
-
- #[test]
- fn url_without_protocol() {
- let url: Url = "https://test.example.com/test/test".to_string().into();
- assert_eq!(url.without_protocol(), "test.example.com/test/test");
-
- let url: Url = "test.example.com/test/test".to_string().into();
- assert_eq!(url.without_protocol(), "test.example.com/test/test");
- }
-
- #[test]
- fn url_is_homepage() {
- let url: Url = "https://test.example.com".to_string().into();
- assert!(url.is_homepage());
-
- let url: Url = "https://test.example.com/".to_string().into();
- assert!(url.is_homepage());
-
- let url: Url = "https://test.example.com/test".to_string().into();
- assert!(!url.is_homepage());
-
- let url: Url = "https://podcasts.apple.com/fr/podcast/beyond-2-cest-quoi-un-planneur-strat%C3%A9gique/id1492683918?i=1000460534325".to_string().into();
- assert!(!url.is_homepage());
-
- let url: Url = "https://podcasts.apple.com".to_string().into();
- assert!(url.is_homepage());
-
- let url: Url = "podcasts.apple.com".to_string().into();
- assert!(url.is_homepage());
- }
-
- #[test]
- fn path_ends_with() {
- let url: Url = "https://test.example.com/test/test".to_string().into();
- assert!(url.path_ends_with("test"));
-
- let url: Url = "https://test.example.com/test/test".to_string().into();
- assert!(url.path_ends_with("test/test"));
-
- let url: Url = "https://test.example.com".to_string().into();
- assert!(!url.path_ends_with("/"));
-
- let url: Url = "https://test.example.zip".to_string().into();
- assert!(!url.path_ends_with(".zip"));
-
- let url: Url = "https://test.example.com".to_string().into();
- assert!(!url.path_ends_with(".com"));
-
- let url: Url = "https://test.example.com/.com".to_string().into();
- assert!(url.path_ends_with(".com"));
-
- let url: Url = "https://test.example.com/test.png".to_string().into();
- assert!(url.path_ends_with(".png"));
- }
-
- #[test]
- fn path_and_query() {
- let url: Url = "https://example.com".to_string().into();
- assert!(url.path_and_query().is_empty());
-
- let url: Url = "https://example.com/".to_string().into();
- assert!(url.path_and_query().is_empty());
-
- let url: Url = "https://example.com/test".to_string().into();
- assert_eq!(url.path_and_query(), "/test");
-
- let url: Url = "https://example.com/test?a=b".to_string().into();
- assert_eq!(url.path_and_query(), "/test?a=b");
-
- let url: Url = "example.com/test?a=b".to_string().into();
- assert_eq!(url.path_and_query(), "/test?a=b");
- }
-
- #[test]
- fn utm_removed_after_normalization() {
- let url: Url = "https://example.com/test?a=b&utm_source=google"
- .to_string()
- .into();
- let url = url.normalize();
- assert_eq!(url.path_and_query(), "/test?a=b");
-
- let url: Url = "https://example.com/test?a=b&utm_source=google&utm_medium=google"
- .to_string()
- .into();
- let url = url.normalize();
- assert_eq!(url.path_and_query(), "/test?a=b");
-
- let url: Url =
- "https://example.com/test?a=b&utm_source=google&utm_medium=google&utm_campaign=google"
- .to_string()
- .into();
- let url = url.normalize();
- assert_eq!(url.path_and_query(), "/test?a=b");
- }
-}