Skip to content

web search done right

Remove www prefix when inserting into web graph

Mikkel Denkerc392f689f9aa
diff --git a/core/src/entrypoint/indexer.rs b/core/src/entrypoint/indexer.rs
index 2b3afc41..1f032137 100644
--- a/core/src/entrypoint/indexer.rs
+++ b/core/src/entrypoint/indexer.rs
@@ -125,7 +125,11 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
})
{
let mut html = Html::parse_without_text(&record.response.body, &record.request.url);
- let node_id = worker.host_centrality_store.node2id.get(&html.url().into());
+ let node = Node::from(html.url());
+ let node_id = worker
+ .host_centrality_store
+ .node2id
+ .get(&node.clone().into_host());
let host_centrality = node_id
.and_then(|node_id| worker.host_centrality_store.harmonic.get(&node_id))
@@ -169,7 +173,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
let mut page_centrality = 0.0;
if let Some(store) = worker.page_centrality_store.as_ref() {
- let node_id = store.node2id.get(&html.url().into());
+ let node_id = store.node2id.get(&node);
page_centrality = node_id
.and_then(|node_id| store.harmonic.get(&node_id))
@@ -186,7 +190,7 @@ pub fn process_job(job: &Job, worker: &IndexingWorker) -> Index {
let node_id = worker
.host_centrality_store
.node2id
- .get(&Node::from_url(html.url()).into_host());
+ .get(&Node::from(html.url()).into_host());
let mut dmoz_description = None;
diff --git a/core/src/webgraph/mod.rs b/core/src/webgraph/mod.rs
index ecc49e2d..9e7a72ca 100644
--- a/core/src/webgraph/mod.rs
+++ b/core/src/webgraph/mod.rs
@@ -57,32 +57,27 @@ impl Node {
pub fn into_host(self) -> Node {
let url = Url::from(self.name);
- let host = url.host_without_specific_subdomains_and_query();
+ let host = url.host_without_www();
Node {
- name: host.to_lowercase(),
+ name: host.to_string(),
}
}
-
- pub fn from_url(url: &Url) -> Self {
- Node::from(url.full())
- }
}
impl From<String> for Node {
fn from(name: String) -> Self {
let url = Url::from(name);
- let name = url.without_protocol();
-
- Self {
- name: name.to_lowercase(),
- }
+ url.into()
}
}
impl From<&Url> for Node {
fn from(url: &Url) -> Self {
- url.full().into()
+ let normalized = url.normalize();
+ Node {
+ name: normalized.into(),
+ }
}
}
@@ -780,7 +775,7 @@ mod test {
fn remove_protocol() {
let n = Node::from("https://www.example.com?test");
- assert_eq!(&n.name, "www.example.com?test");
+ assert_eq!(&n.name, "example.com?test");
}
#[test]
diff --git a/core/src/webpage/url.rs b/core/src/webpage/url.rs
index 9214f473..3790b995 100644
--- a/core/src/webpage/url.rs
+++ b/core/src/webpage/url.rs
@@ -55,7 +55,23 @@ impl From<&str> for Url {
}
}
+impl From<Url> for String {
+ fn from(value: Url) -> Self {
+ value.0
+ }
+}
+
impl Url {
+ pub fn normalize(&self) -> Self {
+ let res: Url = self.strip_protocol().to_lowercase().into();
+
+ if let Some(stripped) = res.0.strip_prefix("www.") {
+ Url(stripped.to_string())
+ } else {
+ res
+ }
+ }
+
pub fn strip_protocol(&self) -> &str {
let mut start_host = 0;
let url = &self.0;
@@ -87,7 +103,13 @@ impl Url {
end_site = url.find('/').expect("The url contains atleast 1 '/'");
}
- &url[..end_site]
+ let res = &url[..end_site];
+
+ if let Some(query_begin) = res.find('?') {
+ &res[..query_begin]
+ } else {
+ res
+ }
}
pub fn domain(&self) -> &str {
@@ -235,21 +257,15 @@ impl Url {
self.full().as_str().parse::<http::Uri>().is_ok()
}
- pub(crate) fn host_without_specific_subdomains_and_query(&self) -> &str {
- let res = if let Some(subdomain) = self.subdomain() {
+ pub(crate) fn host_without_www(&self) -> &str {
+ if let Some(subdomain) = self.subdomain() {
if subdomain == "www" {
self.domain()
} else {
self.site()
}
} else {
- self.site()
- };
-
- if let Some(query_begin) = res.find('?') {
- &res[..query_begin]
- } else {
- res
+ self.domain()
}
}