web search done right
full webgraph should be without protocol
Mikkel Denkerae64cd807a29
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rsindex 271d5a38..673b05ed 100644--- a/src/entrypoint/indexer.rs+++ b/src/entrypoint/indexer.rs@@ -145,7 +145,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {.centrality_store.harmonic.full- .get(&html.url().raw().to_string())+ .get(&html.url().without_protocol().to_lowercase()).unwrap_or_default();let fetch_time_ms = record.metadata.fetch_time_ms as u64;diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rsindex 44b8b2da..08b2ff6a 100644--- a/src/webgraph/mod.rs+++ b/src/webgraph/mod.rs@@ -71,22 +71,20 @@ impl From<String> for Node {impl From<&Url> for Node {fn from(url: &Url) -> Self {Self {- name: url.raw().to_string(),+ name: url.without_protocol().to_lowercase(),}}}impl From<&str> for Node {fn from(name: &str) -> Self {- Self::from(name.to_string())+ Self::from(name.to_lowercase())}}impl From<Url> for Node {fn from(url: Url) -> Self {- Self {- name: url.to_string(),- }+ Self::from(&url)}}diff --git a/src/webpage/url.rs b/src/webpage/url.rsindex a597a62f..dd59dedf 100644--- a/src/webpage/url.rs+++ b/src/webpage/url.rs@@ -222,6 +222,12 @@ impl Url {pub fn matches_url_regex(&self) -> bool {URL_REGEX.is_match(&self.full())}++ pub fn without_protocol(&self) -> &str {+ self.0[self.find_protocol_end()..]+ .strip_prefix("://")+ .unwrap_or_default()+ }}#[cfg(test)]@@ -322,4 +328,11 @@ mod tests {assert_eq!(&url.full_without_id_tags(), "https://test.example.com");}++ #[test]+ fn url_without_protocol() {+ let url: Url = "https://test.example.com/test/test".to_string().into();++ assert_eq!(url.without_protocol(), "test.example.com/test/test");+ }}