web search done right
Remove www prefix when inserting into web graph
Mikkel Denkerc392f689f9aa
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rsindex 2b3afc41..1f032137 100644--- a/core/src/entrypoint/indexer.rs+++ b/core/src/entrypoint/indexer.rs@@ -125,7 +125,11 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {}){let mut html = Html::parse_without_text(&record.response.body, &record.request.url);- let node_id = worker.host_centrality_store.node2id.get(&html.url().into());+ let node = Node::from(html.url());+ let node_id = worker+ .host_centrality_store+ .node2id+ .get(&node.clone().into_host());let host_centrality = node_id.and_then(|node_id| worker.host_centrality_store.harmonic.get(&node_id))@@ -169,7 +173,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {let mut page_centrality = 0.0;if let Some(store) = worker.page_centrality_store.as_ref() {- let node_id = store.node2id.get(&html.url().into());+ let node_id = store.node2id.get(&node);page_centrality = node_id.and_then(|node_id| store.harmonic.get(&node_id))@@ -186,7 +190,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {let node_id = worker.host_centrality_store.node2id- .get(&Node::from_url(html.url()).into_host());+ .get(&Node::from(html.url()).into_host());let mut dmoz_description = None;diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rsindex ecc49e2d..9e7a72ca 100644--- a/core/src/webgraph/mod.rs+++ b/core/src/webgraph/mod.rs@@ -57,32 +57,27 @@ impl Node {pub fn into_host(self) -> Node {let url = Url::from(self.name);- let host = url.host_without_specific_subdomains_and_query();+ let host = url.host_without_www();Node {- name: host.to_lowercase(),+ name: host.to_string(),}}-- pub fn from_url(url: &Url) -> Self {- Node::from(url.full())- }}impl From<String> for Node {fn from(name: String) -> Self {let url = Url::from(name);- let name = url.without_protocol();-- Self {- name: name.to_lowercase(),- }+ url.into()}}impl From<&Url> for Node {fn from(url: &Url) -> Self {- url.full().into()+ let normalized = url.normalize();+ Node {+ name: normalized.into(),+ }}}@@ -780,7 +775,7 @@ mod test {fn remove_protocol() {let n = Node::from("https://www.example.com?test");- assert_eq!(&n.name, "www.example.com?test");+ assert_eq!(&n.name, "example.com?test");}#[test]diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rsindex 9214f473..3790b995 100644--- a/core/src/webpage/url.rs+++ b/core/src/webpage/url.rs@@ -55,7 +55,23 @@ impl From<&str> for Url {}}+impl From<Url> for String {+ fn from(value: Url) -> Self {+ value.0+ }+}+impl Url {+ pub fn normalize(&self) -> Self {+ let res: Url = self.strip_protocol().to_lowercase().into();++ if let Some(stripped) = res.0.strip_prefix("www.") {+ Url(stripped.to_string())+ } else {+ res+ }+ }+pub fn strip_protocol(&self) -> &str {let mut start_host = 0;let url = &self.0;@@ -87,7 +103,13 @@ impl Url {end_site = url.find('/').expect("The url contains atleast 1 '/'");}- &url[..end_site]+ let res = &url[..end_site];++ if let Some(query_begin) = res.find('?') {+ &res[..query_begin]+ } else {+ res+ }}pub fn domain(&self) -> &str {@@ -235,21 +257,15 @@ impl Url {self.full().as_str().parse::<http::Uri>().is_ok()}- pub(crate) fn host_without_specific_subdomains_and_query(&self) -> &str {- let res = if let Some(subdomain) = self.subdomain() {+ pub(crate) fn host_without_www(&self) -> &str {+ if let Some(subdomain) = self.subdomain() {if subdomain == "www" {self.domain()} else {self.site()}} else {- self.site()- };-- if let Some(query_begin) = res.find('?') {- &res[..query_begin]- } else {- res+ self.domain()}}