Skip to content

web search done right

full webgraph should be without protocol

Mikkel Denkerae64cd807a29
diff --git a/src/entrypoint/indexer.rs b/src/entrypoint/indexer.rs
index 271d5a38..673b05ed 100644
--- a/src/entrypoint/indexer.rs
+++ b/src/entrypoint/indexer.rs
@@ -145,7 +145,7 @@ async fn async_process_job(job: &Job, worker: &IndexingWorker) -> Index {
.centrality_store
.harmonic
.full
- .get(&html.url().raw().to_string())
+ .get(&html.url().without_protocol().to_lowercase())
.unwrap_or_default();
let fetch_time_ms = record.metadata.fetch_time_ms as u64;
diff --git a/src/webgraph/mod.rs b/src/webgraph/mod.rs
index 44b8b2da..08b2ff6a 100644
--- a/src/webgraph/mod.rs
+++ b/src/webgraph/mod.rs
@@ -71,22 +71,20 @@ impl From<String> for Node {
impl From<&Url> for Node {
fn from(url: &Url) -> Self {
Self {
- name: url.raw().to_string(),
+ name: url.without_protocol().to_lowercase(),
}
}
}
impl From<&str> for Node {
fn from(name: &str) -> Self {
- Self::from(name.to_string())
+ Self::from(name.to_lowercase())
}
}
impl From<Url> for Node {
fn from(url: Url) -> Self {
- Self {
- name: url.to_string(),
- }
+ Self::from(&url)
}
}
diff --git a/src/webpage/url.rs b/src/webpage/url.rs
index a597a62f..dd59dedf 100644
--- a/src/webpage/url.rs
+++ b/src/webpage/url.rs
@@ -222,6 +222,12 @@ impl Url {
pub fn matches_url_regex(&self) -> bool {
URL_REGEX.is_match(&self.full())
}
+
+ pub fn without_protocol(&self) -> &str {
+ self.0[self.find_protocol_end()..]
+ .strip_prefix("://")
+ .unwrap_or_default()
+ }
}
#[cfg(test)]
@@ -322,4 +328,11 @@ mod tests {
assert_eq!(&url.full_without_id_tags(), "https://test.example.com");
}
+
+ #[test]
+ fn url_without_protocol() {
+ let url: Url = "https://test.example.com/test/test".to_string().into();
+
+ assert_eq!(url.without_protocol(), "test.example.com/test/test");
+ }
}