web search done right
use http instead of https as default protocol
Mikkel Denkercd3a65fe796b
diff --git a/core/src/crawler/crawl_db.rs b/core/src/crawler/crawl_db.rsindex d812ce98..13dbc9ff 100644--- a/core/src/crawler/crawl_db.rs+++ b/core/src/crawler/crawl_db.rs@@ -15,7 +15,7 @@// along with this program. If not, see <https://www.gnu.org/licenses/>.use std::{- collections::{BinaryHeap, HashMap, VecDeque},+ collections::{BTreeMap, BinaryHeap, HashMap, VecDeque},hash::Hash,num::NonZeroUsize,path::Path,@@ -262,7 +262,7 @@ struct DomainState {status: DomainStatus,}-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]pub struct DomainId(u64);impl From<u64> for DomainId {@@ -271,7 +271,7 @@ impl From<u64> for DomainId {}}-#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, PartialOrd, Ord)]struct UrlId(u64);impl From<u64> for UrlId {@@ -311,9 +311,9 @@ pub struct CrawlDb {redirects: RedirectDb,- domain_state: HashMap<DomainId, DomainState>,+ domain_state: BTreeMap<DomainId, DomainState>,- urls: HashMap<DomainId, HashMap<UrlId, UrlState>>,+ urls: BTreeMap<DomainId, BTreeMap<UrlId, UrlState>>,}impl CrawlDb {@@ -326,8 +326,8 @@ impl CrawlDb {url_ids,domain_ids,redirects,- domain_state: HashMap::new(),- urls: HashMap::new(),+ domain_state: BTreeMap::new(),+ urls: BTreeMap::new(),})}diff --git a/core/src/ranking/inbound_similarity.rs b/core/src/ranking/inbound_similarity.rsindex ea138fa5..9cdb3516 100644--- a/core/src/ranking/inbound_similarity.rs+++ b/core/src/ranking/inbound_similarity.rs@@ -343,7 +343,7 @@ mod tests {"#,rand_words(1000)),- "e.com",+ "https://e.com",),backlinks: vec![],host_centrality: 0.0,@@ -371,7 +371,7 @@ mod tests {"#,rand_words(1000)),- "d.com",+ "https://d.com",),backlinks: vec![],host_centrality: 0.01,diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rsindex 55883206..5f17b615 100644--- a/core/src/webpage/url.rs+++ b/core/src/webpage/url.rs@@ -195,11 +195,11 @@ impl Url {pub fn full(&self) -> String {if self.0.starts_with("//") {- return "https:".to_string() + self.0.as_str();+ return "http:".to_string() + self.0.as_str();}if self.find_protocol_end() == 0 {- "https://".to_string() + self.0.as_str()+ "http://".to_string() + self.0.as_str()} else {self.0.clone()}@@ -304,7 +304,7 @@ mod tests {assert_eq!(url.domain(), "dailymail.co.uk");assert_eq!(url.domain_name(), "dailymail");assert_eq!(url.site(), "scripts.dailymail.co.uk");- assert_eq!(url.full(), "https://scripts.dailymail.co.uk");+ assert_eq!(url.full(), "http://scripts.dailymail.co.uk");}#[test]@@ -313,7 +313,7 @@ mod tests {assert_eq!(url.domain(), "dailymail.co.uk");assert_eq!(url.site(), "dailymail.co.uk");- assert_eq!(url.full().as_str(), "https://dailymail.co.uk");+ assert_eq!(url.full().as_str(), "http://dailymail.co.uk");assert!(url.matches_url_regex());}