Skip to content

web search done right

use http instead of https as default protocol

Mikkel Denkercd3a65fe796b
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rs
index d812ce98..13dbc9ff 100644
--- a/core/src/crawler/crawl_db.rs
+++ b/core/src/crawler/crawl_db.rs
@@ -15,7 +15,7 @@
// along with this program. If not, see <https://www.gnu.org/licenses/>.
use std::{
- collections::{BinaryHeap, HashMap, VecDeque},
+ collections::{BTreeMap, BinaryHeap, HashMap, VecDeque},
hash::Hash,
num::NonZeroUsize,
path::Path,
@@ -262,7 +262,7 @@ struct DomainState {
status: DomainStatus,
}
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
pub struct DomainId(u64);
impl From<u64> for DomainId {
@@ -271,7 +271,7 @@ impl From<u64> for DomainId {
}
}
-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]
struct UrlId(u64);
impl From<u64> for UrlId {
@@ -311,9 +311,9 @@ pub struct CrawlDb {
redirects: RedirectDb,
- domain_state: HashMap<DomainId, DomainState>,
+ domain_state: BTreeMap<DomainId, DomainState>,
- urls: HashMap<DomainId, HashMap<UrlId, UrlState>>,
+ urls: BTreeMap<DomainId, BTreeMap<UrlId, UrlState>>,
}
impl CrawlDb {
@@ -326,8 +326,8 @@ impl CrawlDb {
url_ids,
domain_ids,
redirects,
- domain_state: HashMap::new(),
- urls: HashMap::new(),
+ domain_state: BTreeMap::new(),
+ urls: BTreeMap::new(),
})
}
diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rs
index ea138fa5..9cdb3516 100644
--- a/core/src/ranking/inbound_similarity.rs
+++ b/core/src/ranking/inbound_similarity.rs
@@ -343,7 +343,7 @@ mod tests {
"#,
rand_words(1000)
),
- "e.com",
+ "https://e.com",
),
backlinks: vec![],
host_centrality: 0.0,
@@ -371,7 +371,7 @@ mod tests {
"#,
rand_words(1000)
),
- "d.com",
+ "https://d.com",
),
backlinks: vec![],
host_centrality: 0.01,
diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rs
index 55883206..5f17b615 100644
--- a/core/src/webpage/url.rs
+++ b/core/src/webpage/url.rs
@@ -195,11 +195,11 @@ impl Url {
pub fn full(&self) -> String {
if self.0.starts_with("//") {
- return "https:".to_string() + self.0.as_str();
+ return "http:".to_string() + self.0.as_str();
}
if self.find_protocol_end() == 0 {
- "https://".to_string() + self.0.as_str()
+ "http://".to_string() + self.0.as_str()
} else {
self.0.clone()
}
@@ -304,7 +304,7 @@ mod tests {
assert_eq!(url.domain(), "dailymail.co.uk");
assert_eq!(url.domain_name(), "dailymail");
assert_eq!(url.site(), "scripts.dailymail.co.uk");
- assert_eq!(url.full(), "https://scripts.dailymail.co.uk");
+ assert_eq!(url.full(), "http://scripts.dailymail.co.uk");
}
#[test]
@@ -313,7 +313,7 @@ mod tests {
assert_eq!(url.domain(), "dailymail.co.uk");
assert_eq!(url.site(), "dailymail.co.uk");
- assert_eq!(url.full().as_str(), "https://dailymail.co.uk");
+ assert_eq!(url.full().as_str(), "http://dailymail.co.uk");
assert!(url.matches_url_regex());
}